[CCCL] Add missing CCCL components: c2h, nvbench_helper, cmake, cudax, AGENTS.md
Added 863 files from NVIDIA/cccl sparse checkout: - c2h/ (27 files): Catch2 test helpers — generators, validators, runner - nvbench_helper/ (10 files): Benchmark harness utilities - cmake/ (29 files): CMake presets and build helpers - cudax/ (794 files): Experimental CUDA extensions - AGENTS.md: NVIDIA's official AI agent instructions for CCCL - CMakePresets.json: Standardized build configurations - cccl-version.json: Version tracking Also added CCCL_ASSET_MAP.md mapping all 4295 CCCL files to competition value and PRD items. cccl_upstream now covers 100% of competition-critical assets: - 27 tuning headers (SM80/90/100 benchmark data) - 32 dispatch headers (algorithm implementations) - 60 Thrust examples (correctness verification) - 217 CUB Catch2 tests (regression matrix) - 153 CUB benchmarks (parameter space search) - 18 CUB examples (API verification) - 27 test helpers + benchmark harness - 794 cudax experimental extensions
This commit is contained in:
muh-bot
89
cccl_upstream/nvbench_helper/CMakeLists.txt
Normal file
89
cccl_upstream/nvbench_helper/CMakeLists.txt
Normal file
@@ -0,0 +1,89 @@
|
||||
cmake_minimum_required(VERSION 3.21)
|
||||
|
||||
project(nvbench_helper LANGUAGES CXX CUDA)
|
||||
|
||||
include(CMakeDependentOption)
|
||||
|
||||
cccl_get_cub()
|
||||
cccl_get_cudatoolkit()
|
||||
cccl_get_libcudacxx()
|
||||
cccl_get_nvbench()
|
||||
cccl_get_thrust()
|
||||
|
||||
add_library(cccl.nvbench_helper OBJECT nvbench_helper/nvbench_helper.cu)
|
||||
cccl_configure_target(cccl.nvbench_helper)
|
||||
target_link_libraries(
|
||||
cccl.nvbench_helper
|
||||
PUBLIC
|
||||
libcudacxx::libcudacxx
|
||||
CUB::CUB
|
||||
# This Thrust target does not define any host/device system. Those are added later if needed:
|
||||
Thrust::Thrust
|
||||
nvbench::nvbench
|
||||
PRIVATE #
|
||||
cccl.compiler_interface
|
||||
CUDA::curand
|
||||
)
|
||||
|
||||
target_include_directories(
|
||||
cccl.nvbench_helper
|
||||
PUBLIC "${CMAKE_CURRENT_LIST_DIR}/nvbench_helper"
|
||||
)
|
||||
|
||||
# Old NVCC incorrectly infers the host device annotation of std::map::map() which calls a non-host device member
|
||||
if (CUDAToolkit_VERSION VERSION_LESS "12.5.0")
|
||||
target_compile_options(cccl.nvbench_helper PUBLIC -diag-suppress 20011)
|
||||
endif()
|
||||
|
||||
if (CCCL_ENABLE_TILE)
|
||||
target_compile_options(
|
||||
cccl.nvbench_helper
|
||||
PUBLIC "$<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:--enable-tile>"
|
||||
)
|
||||
endif()
|
||||
|
||||
# Don't enable tests when pulled in as an internal dependency.
|
||||
# Require CCCL_ENABLE_NVBENCH_HELPER to be explicitly enabled.
|
||||
cmake_dependent_option(
|
||||
nvbench_helper_ENABLE_TESTING
|
||||
"Enable tests for nvbench_helper"
|
||||
ON
|
||||
CCCL_ENABLE_NVBENCH_HELPER
|
||||
OFF
|
||||
)
|
||||
mark_as_advanced(nvbench_helper_ENABLE_TESTING)
|
||||
|
||||
if (nvbench_helper_ENABLE_TESTING)
|
||||
cccl_get_catch2()
|
||||
cccl_get_boost()
|
||||
|
||||
thrust_create_target(cccl.nvbench_helper.test.Thrust.cpp.cuda DEVICE CUDA)
|
||||
thrust_create_target(cccl.nvbench_helper.test.Thrust.cpp.cpp DEVICE CPP)
|
||||
|
||||
function(add_nvbench_helper_test device_system)
|
||||
set(nvbench_helper_test_target nvbench_helper.test.${device_system})
|
||||
cccl_add_executable(
|
||||
${nvbench_helper_test_target}
|
||||
ADD_CTEST
|
||||
NO_METATARGETS
|
||||
SOURCES
|
||||
test/gen_seed.cu
|
||||
test/gen_range.cu
|
||||
test/gen_entropy.cu
|
||||
test/gen_uniform_distribution.cu
|
||||
test/gen_power_law_distribution.cu
|
||||
)
|
||||
target_link_libraries(
|
||||
${nvbench_helper_test_target}
|
||||
PRIVATE
|
||||
cccl.compiler_interface
|
||||
cccl.nvbench_helper
|
||||
cccl.nvbench_helper.test.Thrust.cpp.${device_system}
|
||||
Catch2::Catch2WithMain
|
||||
Boost::math
|
||||
)
|
||||
endfunction()
|
||||
|
||||
add_nvbench_helper_test(cpp)
|
||||
add_nvbench_helper_test(cuda)
|
||||
endif()
|
||||
@@ -0,0 +1,124 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cuda/cmath>
|
||||
#include <cuda/ptx>
|
||||
#include <cuda/std/cstdint>
|
||||
#include <cuda/std/cstring>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/utility>
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ static T generate_random_data()
|
||||
{
|
||||
constexpr auto size = static_cast<int>(cuda::ceil_div(sizeof(T), sizeof(uint32_t)));
|
||||
uint32_t data[size];
|
||||
for (int i = 0; i < size; i++)
|
||||
{
|
||||
data[i] = cuda::ptx::get_sreg_clock();
|
||||
}
|
||||
T ret;
|
||||
::cuda::std::memcpy(&ret, data, sizeof(T));
|
||||
return ret;
|
||||
}
|
||||
|
||||
// When benchmarking algorithms sensitive to data distribution, each thread should use a different seed to get different
|
||||
// data.
|
||||
template <typename T>
|
||||
__device__ __forceinline__ static T generate_random_data(uint32_t& seed)
|
||||
{
|
||||
constexpr auto size = static_cast<int>(cuda::ceil_div(sizeof(T), sizeof(uint32_t)));
|
||||
uint32_t data[size];
|
||||
for (int i = 0; i < size; i++)
|
||||
{
|
||||
// https://en.wikipedia.org/wiki/Linear_congruential_generator
|
||||
seed = 1664525 * seed + 1013904223;
|
||||
data[i] = seed;
|
||||
}
|
||||
T ret;
|
||||
cuda::std::memcpy(&ret, data, sizeof(T));
|
||||
return ret;
|
||||
}
|
||||
|
||||
__device__ static int device_var[16];
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ static void sink(T value)
|
||||
{
|
||||
if (cuda::ptx::get_sreg_smid() == static_cast<uint32_t>(-1))
|
||||
{
|
||||
*reinterpret_cast<T*>(device_var) = value;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, int Size>
|
||||
__device__ __forceinline__ static void sink(T (&values)[Size])
|
||||
{
|
||||
if (cuda::ptx::get_sreg_smid() == static_cast<uint32_t>(-1))
|
||||
{
|
||||
// use float instead of T to ensure summation order matters (due to floating-point non-associativity), making
|
||||
// `action` less likely to be optimized away
|
||||
float sum(0.0f);
|
||||
for (int i = 0; i < Size; ++i)
|
||||
{
|
||||
sum += values[i];
|
||||
}
|
||||
*reinterpret_cast<float*>(device_var) += sum;
|
||||
}
|
||||
}
|
||||
|
||||
template <int ThreadsPerBlock, int UnrollFactor, typename ActionT, typename T>
|
||||
__launch_bounds__(ThreadsPerBlock) __global__ static void benchmark_kernel(const ActionT action)
|
||||
{
|
||||
auto data = generate_random_data<T>();
|
||||
cuda::static_for<UnrollFactor>([&]([[maybe_unused]] auto _) {
|
||||
data = action(data);
|
||||
});
|
||||
sink(data);
|
||||
}
|
||||
|
||||
// This variant uses pragma directive to prevent loop unrolling, which can cause high register pressure and skew
|
||||
// benchmark results.
|
||||
// For keys-only benchmarks, set ValueT to void.
|
||||
template <int ItemsPerThread, typename KeyT, typename ValueT, typename ActionT, typename... Args>
|
||||
__global__ static void benchmark_kernel(int num_iterations, const ActionT action, Args... args)
|
||||
{
|
||||
constexpr int warp_threads = 32;
|
||||
constexpr bool has_values = !cuda::std::is_void_v<ValueT>;
|
||||
KeyT keys[ItemsPerThread];
|
||||
// when ValueT=void, declare values as char array
|
||||
[[maybe_unused]] cuda::std::conditional_t<has_values, ValueT, char> values[ItemsPerThread];
|
||||
const auto tid = threadIdx.x;
|
||||
// Shift tid by 7 to reduce the likelihood of threads within a warp getting monotonically increasing data
|
||||
uint32_t seed = cuda::ptx::get_sreg_clock() + (tid + 7) % warp_threads;
|
||||
|
||||
#pragma unroll 1
|
||||
for (int iter = 0; iter < num_iterations; ++iter)
|
||||
{
|
||||
for (int i = 0; i < ItemsPerThread; ++i)
|
||||
{
|
||||
keys[i] = generate_random_data<KeyT>(seed);
|
||||
if constexpr (has_values)
|
||||
{
|
||||
values[i] = generate_random_data<ValueT>(seed);
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (has_values)
|
||||
{
|
||||
action(keys, values, args...);
|
||||
}
|
||||
else
|
||||
{
|
||||
action(keys, args...);
|
||||
}
|
||||
|
||||
sink(keys);
|
||||
if constexpr (has_values)
|
||||
{
|
||||
sink(values);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2011-2026, NVIDIA CORPORATION. All rights reserved.
|
||||
// SPDX-License-Identifier: BSD-3
|
||||
|
||||
#pragma once
|
||||
|
||||
#if !TUNE_BASE
|
||||
# include <cub/detail/delay_constructor.cuh>
|
||||
|
||||
# include <nvbench_helper.cuh>
|
||||
|
||||
# if !defined(TUNE_MAGIC_NS) || !defined(TUNE_L2_WRITE_LATENCY_NS) || !defined(TUNE_DELAY_CONSTRUCTOR_ID)
|
||||
# error "TUNE_MAGIC_NS, TUNE_L2_WRITE_LATENCY_NS, and TUNE_DELAY_CONSTRUCTOR_ID must be defined"
|
||||
# endif
|
||||
|
||||
using delay_constructor_t =
|
||||
cub::detail::delay_constructor_t<static_cast<cub::LookbackDelayAlgorithm>(TUNE_DELAY_CONSTRUCTOR_ID),
|
||||
TUNE_MAGIC_NS,
|
||||
TUNE_L2_WRITE_LATENCY_NS>;
|
||||
|
||||
inline constexpr auto lookback_delay_policy = cub::LookbackDelayPolicy{
|
||||
static_cast<cub::LookbackDelayAlgorithm>(TUNE_DELAY_CONSTRUCTOR_ID), TUNE_MAGIC_NS, TUNE_L2_WRITE_LATENCY_NS};
|
||||
|
||||
#endif // !TUNE_BASE
|
||||
840
cccl_upstream/nvbench_helper/nvbench_helper/nvbench_helper.cu
Normal file
840
cccl_upstream/nvbench_helper/nvbench_helper/nvbench_helper.cu
Normal file
@@ -0,0 +1,840 @@
|
||||
#include <cub/device/device_copy.cuh>
|
||||
|
||||
#include <thrust/binary_search.h>
|
||||
#include <thrust/count.h>
|
||||
#include <thrust/detail/raw_pointer_cast.h>
|
||||
#include <thrust/distance.h>
|
||||
#include <thrust/execution_policy.h>
|
||||
#include <thrust/fill.h>
|
||||
#include <thrust/for_each.h>
|
||||
#include <thrust/host_vector.h>
|
||||
#include <thrust/iterator/counting_iterator.h>
|
||||
#include <thrust/iterator/transform_output_iterator.h>
|
||||
#include <thrust/iterator/zip_iterator.h>
|
||||
#include <thrust/scan.h>
|
||||
#include <thrust/tabulate.h>
|
||||
|
||||
#include <cuda/functional>
|
||||
#include <cuda/iterator>
|
||||
#include <cuda/std/__floating_point/cuda_fp_types.h> // __half, __nv_bfloat16
|
||||
#include <cuda/std/bit>
|
||||
#include <cuda/type_traits>
|
||||
|
||||
#include <cstdint>
|
||||
#include <random>
|
||||
#include <type_traits>
|
||||
|
||||
#include <curand.h>
|
||||
#include <nvbench_helper.cuh>
|
||||
|
||||
#include "thrust/device_vector.h"
|
||||
|
||||
namespace detail
|
||||
{
|
||||
constexpr double lognormal_mean = 3.0;
|
||||
constexpr double lognormal_sigma = 1.2;
|
||||
|
||||
enum class executor
|
||||
{
|
||||
host,
|
||||
device
|
||||
};
|
||||
|
||||
class host_generator_t
|
||||
{
|
||||
public:
|
||||
template <typename T>
|
||||
void generate(seed_t seed, cuda::std::span<T> device_span, bit_entropy entropy, T min, T max);
|
||||
|
||||
const double* new_uniform_distribution(seed_t seed, std::size_t num_items);
|
||||
const double* new_lognormal_distribution(seed_t seed, std::size_t num_items);
|
||||
const double* new_constant(std::size_t num_items, double val);
|
||||
|
||||
private:
|
||||
thrust::host_vector<double> m_distribution;
|
||||
};
|
||||
|
||||
const double* host_generator_t::new_uniform_distribution(seed_t seed, std::size_t num_items)
|
||||
{
|
||||
m_distribution.resize(num_items);
|
||||
double* h_distribution = thrust::raw_pointer_cast(m_distribution.data());
|
||||
|
||||
std::default_random_engine re(seed.get());
|
||||
std::uniform_real_distribution<double> dist(0.0, 1.0);
|
||||
|
||||
for (std::size_t i = 0; i < num_items; i++)
|
||||
{
|
||||
h_distribution[i] = dist(re);
|
||||
}
|
||||
|
||||
return h_distribution;
|
||||
}
|
||||
|
||||
const double* host_generator_t::new_lognormal_distribution(seed_t seed, std::size_t num_items)
|
||||
{
|
||||
m_distribution.resize(num_items);
|
||||
double* h_distribution = thrust::raw_pointer_cast(m_distribution.data());
|
||||
|
||||
std::default_random_engine re(seed.get());
|
||||
std::lognormal_distribution<double> dist(lognormal_mean, lognormal_sigma);
|
||||
|
||||
for (std::size_t i = 0; i < num_items; i++)
|
||||
{
|
||||
h_distribution[i] = dist(re);
|
||||
}
|
||||
|
||||
return h_distribution;
|
||||
}
|
||||
|
||||
const double* host_generator_t::new_constant(std::size_t num_items, double val)
|
||||
{
|
||||
m_distribution.resize(num_items);
|
||||
double* h_distribution = thrust::raw_pointer_cast(m_distribution.data());
|
||||
thrust::fill_n(thrust::host, h_distribution, num_items, val);
|
||||
return h_distribution;
|
||||
}
|
||||
|
||||
class device_generator_t
|
||||
{
|
||||
public:
|
||||
device_generator_t()
|
||||
{
|
||||
curandCreateGenerator(&m_gen, CURAND_RNG_PSEUDO_DEFAULT);
|
||||
}
|
||||
|
||||
~device_generator_t()
|
||||
{
|
||||
curandDestroyGenerator(m_gen);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void generate(seed_t seed, cuda::std::span<T> device_span, bit_entropy entropy, T min, T max);
|
||||
|
||||
const double* new_uniform_distribution(seed_t seed, std::size_t num_items);
|
||||
const double* new_lognormal_distribution(seed_t seed, std::size_t num_items);
|
||||
const double* new_constant(std::size_t num_items, double val);
|
||||
|
||||
private:
|
||||
curandGenerator_t m_gen;
|
||||
thrust::device_vector<double> m_distribution;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct random_to_item_t
|
||||
{
|
||||
double m_min;
|
||||
double m_max;
|
||||
|
||||
__host__ __device__ random_to_item_t(T min, T max)
|
||||
: m_min(static_cast<double>(min))
|
||||
, m_max(static_cast<double>(max))
|
||||
{}
|
||||
|
||||
__host__ __device__ T operator()(double random_value) const
|
||||
{
|
||||
if constexpr (::cuda::is_floating_point_v<T>)
|
||||
{
|
||||
return static_cast<T>((m_max - m_min) * random_value + m_min);
|
||||
}
|
||||
else
|
||||
{
|
||||
return static_cast<T>(floor((m_max - m_min + 1) * random_value + m_min));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const double* device_generator_t::new_uniform_distribution(seed_t seed, std::size_t num_items)
|
||||
{
|
||||
m_distribution.resize(num_items);
|
||||
double* d_distribution = thrust::raw_pointer_cast(m_distribution.data());
|
||||
|
||||
curandSetPseudoRandomGeneratorSeed(m_gen, seed.get());
|
||||
curandGenerateUniformDouble(m_gen, d_distribution, num_items);
|
||||
|
||||
return d_distribution;
|
||||
}
|
||||
|
||||
const double* device_generator_t::new_lognormal_distribution(seed_t seed, std::size_t num_items)
|
||||
{
|
||||
m_distribution.resize(num_items);
|
||||
double* d_distribution = thrust::raw_pointer_cast(m_distribution.data());
|
||||
|
||||
curandSetPseudoRandomGeneratorSeed(m_gen, seed.get());
|
||||
curandGenerateLogNormalDouble(m_gen, d_distribution, num_items, lognormal_mean, lognormal_sigma);
|
||||
|
||||
return d_distribution;
|
||||
}
|
||||
|
||||
const double* device_generator_t::new_constant(std::size_t num_items, double val)
|
||||
{
|
||||
m_distribution.resize(num_items);
|
||||
double* d_distribution = thrust::raw_pointer_cast(m_distribution.data());
|
||||
thrust::fill_n(thrust::device, d_distribution, num_items, val);
|
||||
return d_distribution;
|
||||
}
|
||||
|
||||
struct and_t
|
||||
{
|
||||
template <class T>
|
||||
__host__ __device__ T operator()(T a, T b) const
|
||||
{
|
||||
return a & b;
|
||||
}
|
||||
|
||||
__host__ __device__ float operator()(float a, float b) const
|
||||
{
|
||||
const std::uint32_t result = cuda::std::bit_cast<std::uint32_t>(a) & cuda::std::bit_cast<std::uint32_t>(b);
|
||||
return cuda::std::bit_cast<float>(result);
|
||||
}
|
||||
|
||||
__host__ __device__ double operator()(double a, double b) const
|
||||
{
|
||||
const std::uint64_t result = cuda::std::bit_cast<std::uint64_t>(a) & cuda::std::bit_cast<std::uint64_t>(b);
|
||||
return cuda::std::bit_cast<double>(result);
|
||||
}
|
||||
|
||||
#if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
|
||||
__host__ __device__ __half operator()(__half a, __half b) const
|
||||
{
|
||||
const std::uint16_t result = cuda::std::bit_cast<std::uint16_t>(a) & cuda::std::bit_cast<std::uint16_t>(b);
|
||||
return cuda::std::bit_cast<__half>(result);
|
||||
}
|
||||
#endif // _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
|
||||
|
||||
#if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
|
||||
__host__ __device__ __nv_bfloat16 operator()(__nv_bfloat16 a, __nv_bfloat16 b) const
|
||||
{
|
||||
const std::uint16_t result = cuda::std::bit_cast<std::uint16_t>(a) & cuda::std::bit_cast<std::uint16_t>(b);
|
||||
return cuda::std::bit_cast<__nv_bfloat16>(result);
|
||||
}
|
||||
#endif // _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
|
||||
|
||||
template <typename T>
|
||||
__host__ __device__ cuda::std::complex<T> operator()(cuda::std::complex<T> a, cuda::std::complex<T> b) const
|
||||
{
|
||||
const T a_real = a.real();
|
||||
const T a_imag = a.imag();
|
||||
|
||||
const T b_real = b.real();
|
||||
const T b_imag = b.imag();
|
||||
|
||||
using uint_t = std::conditional_t<sizeof(T) == 4, std::uint32_t, std::uint64_t>;
|
||||
const auto result_real = cuda::std::bit_cast<uint_t>(a_real) & cuda::std::bit_cast<uint_t>(b_real);
|
||||
const auto result_imag = cuda::std::bit_cast<uint_t>(a_imag) & cuda::std::bit_cast<uint_t>(b_imag);
|
||||
|
||||
return {cuda::std::bit_cast<T>(result_real), cuda::std::bit_cast<T>(result_imag)};
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct set_real_t
|
||||
{
|
||||
cuda::std::complex<T> m_min{};
|
||||
cuda::std::complex<T> m_max{};
|
||||
cuda::std::complex<T>* m_d_in{};
|
||||
const double* m_d_tmp{};
|
||||
|
||||
__host__ __device__ void operator()(std::size_t i) const
|
||||
{
|
||||
m_d_in[i].real(random_to_item_t<double>{m_min.real(), m_max.real()}(m_d_tmp[i]));
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct set_imag_t
|
||||
{
|
||||
cuda::std::complex<T> m_min{};
|
||||
cuda::std::complex<T> m_max{};
|
||||
cuda::std::complex<T>* m_d_in{};
|
||||
const double* m_d_tmp{};
|
||||
|
||||
__host__ __device__ void operator()(std::size_t i) const
|
||||
{
|
||||
m_d_in[i].imag(random_to_item_t<double>{m_min.imag(), m_max.imag()}(m_d_tmp[i]));
|
||||
}
|
||||
};
|
||||
|
||||
template <class T>
|
||||
struct lognormal_transformer_t
|
||||
{
|
||||
std::size_t total_elements;
|
||||
double sum;
|
||||
|
||||
__host__ __device__ T operator()(double val) const
|
||||
{
|
||||
return floor(val * total_elements / sum);
|
||||
}
|
||||
};
|
||||
|
||||
class generator_t
|
||||
{
|
||||
public:
|
||||
template <typename T>
|
||||
void generate(executor exec, seed_t seed, cuda::std::span<T> span, bit_entropy entropy, T min, T max)
|
||||
{
|
||||
construct_guard(exec);
|
||||
|
||||
if (exec == executor::device)
|
||||
{
|
||||
this->generate(thrust::device, *m_device_generator, seed, span, entropy, min, max);
|
||||
}
|
||||
else
|
||||
{
|
||||
this->generate(thrust::host, *m_host_generator, seed, span, entropy, min, max);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void power_law_segment_offsets(executor exec, seed_t seed, cuda::std::span<T> span, std::size_t total_elements)
|
||||
{
|
||||
construct_guard(exec);
|
||||
|
||||
if (exec == executor::device)
|
||||
{
|
||||
this->power_law_segment_offsets(thrust::device, *m_device_generator, seed, span, total_elements);
|
||||
}
|
||||
else
|
||||
{
|
||||
this->power_law_segment_offsets(thrust::host, *m_host_generator, seed, span, total_elements);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
void construct_guard(executor exec)
|
||||
{
|
||||
if (exec == executor::device)
|
||||
{
|
||||
if (!m_device_generator)
|
||||
{
|
||||
m_device_generator.emplace();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!m_host_generator)
|
||||
{
|
||||
m_host_generator.emplace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename ExecT, typename DistT, typename T>
|
||||
void generate(const ExecT& exec, DistT& dist, seed_t seed, cuda::std::span<T> span, bit_entropy entropy, T min, T max);
|
||||
|
||||
template <typename ExecT, typename DistT, typename T>
|
||||
void generate(const ExecT& exec,
|
||||
DistT& dist,
|
||||
seed_t seed,
|
||||
cuda::std::span<cuda::std::complex<T>> span,
|
||||
bit_entropy entropy,
|
||||
cuda::std::complex<T> min,
|
||||
cuda::std::complex<T> max);
|
||||
|
||||
template <typename ExecT, typename DistT>
|
||||
void generate(
|
||||
const ExecT& exec, DistT& dist, seed_t seed, cuda::std::span<bool> span, bit_entropy entropy, bool min, bool max);
|
||||
|
||||
template <typename ExecT, typename DistT, typename T>
|
||||
void power_law_segment_offsets(
|
||||
const ExecT& exec, DistT& dist, seed_t seed, cuda::std::span<T> span, std::size_t total_elements);
|
||||
|
||||
std::optional<host_generator_t> m_host_generator;
|
||||
std::optional<device_generator_t> m_device_generator;
|
||||
};
|
||||
|
||||
template <typename ExecT, typename DistT, typename T>
|
||||
void generator_t::generate(
|
||||
const ExecT& exec, DistT& dist, seed_t seed, cuda::std::span<T> span, bit_entropy entropy, T min, T max)
|
||||
{
|
||||
switch (entropy)
|
||||
{
|
||||
case bit_entropy::_1_000: {
|
||||
const double* uniform_distribution = dist.new_uniform_distribution(seed, span.size());
|
||||
|
||||
thrust::transform(
|
||||
exec,
|
||||
uniform_distribution,
|
||||
uniform_distribution + span.size(),
|
||||
span.data(),
|
||||
::cuda::proclaim_copyable_arguments(random_to_item_t<T>(min, max)));
|
||||
return;
|
||||
}
|
||||
case bit_entropy::_0_000: {
|
||||
std::mt19937 rng;
|
||||
rng.seed(static_cast<std::mt19937::result_type>(seed.get()));
|
||||
std::uniform_real_distribution<float> dist(0.0f, 1.0f);
|
||||
T random_value = random_to_item_t<T>(min, max)(dist(rng));
|
||||
thrust::fill(exec, span.data(), span.data() + span.size(), random_value);
|
||||
return;
|
||||
}
|
||||
default: {
|
||||
const double* uniform_distribution = dist.new_uniform_distribution(seed, span.size());
|
||||
++seed;
|
||||
|
||||
thrust::transform(
|
||||
exec,
|
||||
uniform_distribution,
|
||||
uniform_distribution + span.size(),
|
||||
span.data(),
|
||||
::cuda::proclaim_copyable_arguments(random_to_item_t<T>(min, max)));
|
||||
|
||||
const int number_of_steps = static_cast<int>(entropy);
|
||||
|
||||
constexpr bool is_device = std::is_same_v<DistT, device_generator_t>;
|
||||
using vec_t = std::conditional_t<is_device, thrust::device_vector<T>, thrust::host_vector<T>>;
|
||||
vec_t tmp_vec(span.size());
|
||||
cuda::std::span<T> tmp(thrust::raw_pointer_cast(tmp_vec.data()), tmp_vec.size());
|
||||
|
||||
for (int i = 0; i < number_of_steps; i++, ++seed)
|
||||
{
|
||||
this->generate(is_device ? executor::device : executor::host, seed, tmp, bit_entropy::_1_000, min, max);
|
||||
|
||||
thrust::transform(
|
||||
exec,
|
||||
span.data(),
|
||||
span.data() + span.size(),
|
||||
tmp.data(),
|
||||
span.data(),
|
||||
::cuda::proclaim_copyable_arguments(and_t{}));
|
||||
}
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
template <typename ExecT, typename DistT, typename T>
|
||||
void generator_t::generate(
|
||||
const ExecT& exec,
|
||||
DistT& dist,
|
||||
seed_t seed,
|
||||
cuda::std::span<cuda::std::complex<T>> span,
|
||||
bit_entropy entropy,
|
||||
cuda::std::complex<T> min,
|
||||
cuda::std::complex<T> max)
|
||||
{
|
||||
switch (entropy)
|
||||
{
|
||||
case bit_entropy::_1_000: {
|
||||
const double* uniform_distribution = dist.new_uniform_distribution(seed, span.size());
|
||||
thrust::for_each_n(exec,
|
||||
thrust::make_counting_iterator(std::size_t{0}),
|
||||
span.size(),
|
||||
set_real_t<T>{min, max, span.data(), uniform_distribution});
|
||||
++seed;
|
||||
|
||||
uniform_distribution = dist.new_uniform_distribution(seed, span.size());
|
||||
thrust::for_each_n(exec,
|
||||
thrust::make_counting_iterator(std::size_t{0}),
|
||||
span.size(),
|
||||
set_imag_t<T>{min, max, span.data(), uniform_distribution});
|
||||
++seed;
|
||||
return;
|
||||
}
|
||||
case bit_entropy::_0_000: {
|
||||
std::mt19937 rng;
|
||||
rng.seed(static_cast<std::mt19937::result_type>(seed.get()));
|
||||
std::uniform_real_distribution<double> dist(0.0f, 1.0f);
|
||||
const float random_imag = random_to_item_t<double>(min.imag(), max.imag())(dist(rng));
|
||||
const float random_real = random_to_item_t<double>(min.imag(), max.imag())(dist(rng));
|
||||
thrust::fill(exec, span.data(), span.data() + span.size(), cuda::std::complex<T>{random_real, random_imag});
|
||||
return;
|
||||
}
|
||||
default: {
|
||||
const double* uniform_distribution = dist.new_uniform_distribution(seed, span.size());
|
||||
thrust::for_each_n(exec,
|
||||
thrust::make_counting_iterator(std::size_t{0}),
|
||||
span.size(),
|
||||
set_real_t<T>{min, max, span.data(), uniform_distribution});
|
||||
++seed;
|
||||
|
||||
uniform_distribution = dist.new_uniform_distribution(seed, span.size());
|
||||
thrust::for_each_n(exec,
|
||||
thrust::make_counting_iterator(std::size_t{0}),
|
||||
span.size(),
|
||||
set_imag_t<T>{min, max, span.data(), uniform_distribution});
|
||||
++seed;
|
||||
|
||||
const int number_of_steps = static_cast<int>(entropy);
|
||||
|
||||
constexpr bool is_device = std::is_same_v<DistT, device_generator_t>;
|
||||
using vec_t = std::conditional_t<is_device,
|
||||
thrust::device_vector<cuda::std::complex<T>>,
|
||||
thrust::host_vector<cuda::std::complex<T>>>;
|
||||
|
||||
vec_t tmp_vec(span.size());
|
||||
cuda::std::span<cuda::std::complex<T>> tmp(thrust::raw_pointer_cast(tmp_vec.data()), tmp_vec.size());
|
||||
|
||||
for (int i = 0; i < number_of_steps; i++, ++seed)
|
||||
{
|
||||
this->generate(is_device ? executor::device : executor::host, seed, tmp, bit_entropy::_1_000, min, max);
|
||||
|
||||
thrust::transform(
|
||||
exec,
|
||||
span.data(),
|
||||
span.data() + span.size(),
|
||||
tmp.data(),
|
||||
span.data(),
|
||||
::cuda::proclaim_copyable_arguments(and_t{}));
|
||||
}
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
struct random_to_probability_t
|
||||
{
|
||||
double m_probability;
|
||||
|
||||
__host__ __device__ bool operator()(double random_value) const
|
||||
{
|
||||
return random_value < m_probability;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename ExecT, typename DistT>
|
||||
void generator_t::generate(
|
||||
const ExecT& exec,
|
||||
DistT& dist,
|
||||
seed_t seed,
|
||||
cuda::std::span<bool> span,
|
||||
bit_entropy entropy,
|
||||
bool /* min */,
|
||||
bool /* max */)
|
||||
{
|
||||
if (entropy == bit_entropy::_0_000)
|
||||
{
|
||||
thrust::fill(exec, span.data(), span.data() + span.size(), false);
|
||||
}
|
||||
else if (entropy == bit_entropy::_1_000)
|
||||
{
|
||||
thrust::fill(exec, span.data(), span.data() + span.size(), true);
|
||||
}
|
||||
else
|
||||
{
|
||||
const double* uniform_distribution = dist.new_uniform_distribution(seed, span.size());
|
||||
|
||||
thrust::transform(
|
||||
exec,
|
||||
uniform_distribution,
|
||||
uniform_distribution + span.size(),
|
||||
span.data(),
|
||||
::cuda::proclaim_copyable_arguments(random_to_probability_t{entropy_to_probability(entropy)}));
|
||||
}
|
||||
}
|
||||
|
||||
template <class T>
|
||||
struct lognormal_adjust_t
|
||||
{
|
||||
T* segment_sizes{};
|
||||
|
||||
__host__ __device__ T operator()(std::size_t sid) const
|
||||
{
|
||||
return segment_sizes[sid] + 1;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename ExecT, typename DistT, typename T>
|
||||
void generator_t::power_law_segment_offsets(
|
||||
const ExecT& exec, DistT& dist, seed_t seed, cuda::std::span<T> device_segment_offsets, std::size_t total_elements)
|
||||
{
|
||||
const std::size_t total_segments = device_segment_offsets.size() - 1;
|
||||
const double* uniform_distribution = dist.new_lognormal_distribution(seed, total_segments);
|
||||
|
||||
if (static_cast<std::size_t>(thrust::count(exec, uniform_distribution, uniform_distribution + total_segments, 0.0))
|
||||
== total_segments)
|
||||
{
|
||||
uniform_distribution = dist.new_constant(total_segments, 1.0);
|
||||
}
|
||||
|
||||
const double sum = thrust::reduce(exec, uniform_distribution, uniform_distribution + total_segments);
|
||||
|
||||
thrust::transform(
|
||||
exec,
|
||||
uniform_distribution,
|
||||
uniform_distribution + total_segments,
|
||||
device_segment_offsets.data(),
|
||||
::cuda::proclaim_copyable_arguments(lognormal_transformer_t<T>{total_elements, sum}));
|
||||
|
||||
const int diff =
|
||||
total_elements
|
||||
- thrust::reduce(exec, device_segment_offsets.data(), device_segment_offsets.data() + device_segment_offsets.size());
|
||||
|
||||
if (diff > 0)
|
||||
{
|
||||
thrust::tabulate(exec,
|
||||
device_segment_offsets.data(),
|
||||
device_segment_offsets.data() + diff,
|
||||
lognormal_adjust_t<T>{device_segment_offsets.data()});
|
||||
}
|
||||
|
||||
thrust::exclusive_scan(
|
||||
exec,
|
||||
device_segment_offsets.data(),
|
||||
device_segment_offsets.data() + device_segment_offsets.size(),
|
||||
device_segment_offsets.data());
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void gen(executor exec, seed_t seed, cuda::std::span<T> span, bit_entropy entropy, T min, T max)
|
||||
{
|
||||
generator_t{}.generate(exec, seed, span, entropy, min, max);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void gen_host(seed_t seed, cuda::std::span<T> span, bit_entropy entropy, T min, T max)
|
||||
{
|
||||
gen(executor::host, seed, span, entropy, min, max);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void gen_device(seed_t seed, cuda::std::span<T> device_span, bit_entropy entropy, T min, T max)
|
||||
{
|
||||
gen(executor::device, seed, device_span, entropy, min, max);
|
||||
}
|
||||
|
||||
template <class T>
|
||||
struct offset_to_iterator_t
|
||||
{
|
||||
T* base_it;
|
||||
|
||||
__host__ __device__ __forceinline__ T* operator()(std::size_t offset) const
|
||||
{
|
||||
return base_it + offset;
|
||||
}
|
||||
};
|
||||
|
||||
template <class T>
|
||||
struct repeat_index_t
|
||||
{
|
||||
__host__ __device__ __forceinline__ cuda::constant_iterator<T> operator()(std::size_t i)
|
||||
{
|
||||
return cuda::constant_iterator<T>(static_cast<T>(i));
|
||||
}
|
||||
};
|
||||
|
||||
struct offset_to_size_t
|
||||
{
|
||||
std::size_t* offsets = nullptr;
|
||||
|
||||
__host__ __device__ __forceinline__ std::size_t operator()(std::size_t i)
|
||||
{
|
||||
return offsets[i + 1] - offsets[i];
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
void gen_key_segments(executor exec, seed_t, cuda::std::span<T> keys, cuda::std::span<std::size_t> segment_offsets)
|
||||
{
|
||||
thrust::counting_iterator<int> iota(0);
|
||||
offset_to_iterator_t<T> dst_transform_op{keys.data()};
|
||||
|
||||
const std::size_t total_segments = segment_offsets.size() - 1;
|
||||
|
||||
auto d_range_srcs = thrust::make_transform_iterator(iota, repeat_index_t<T>{});
|
||||
auto d_range_dsts = thrust::make_transform_iterator(segment_offsets.data(), dst_transform_op);
|
||||
auto d_range_sizes = thrust::make_transform_iterator(iota, offset_to_size_t{segment_offsets.data()});
|
||||
|
||||
if (exec == executor::device)
|
||||
{
|
||||
std::uint8_t* d_temp_storage = nullptr;
|
||||
std::size_t temp_storage_bytes = 0;
|
||||
cub::DeviceCopy::Batched(
|
||||
d_temp_storage, temp_storage_bytes, d_range_srcs, d_range_dsts, d_range_sizes, total_segments);
|
||||
|
||||
thrust::device_vector<std::uint8_t> temp_storage(temp_storage_bytes);
|
||||
d_temp_storage = thrust::raw_pointer_cast(temp_storage.data());
|
||||
|
||||
cub::DeviceCopy::Batched(
|
||||
d_temp_storage, temp_storage_bytes, d_range_srcs, d_range_dsts, d_range_sizes, total_segments);
|
||||
cudaDeviceSynchronize();
|
||||
}
|
||||
else
|
||||
{
|
||||
for (std::size_t sid = 0; sid < total_segments; sid++)
|
||||
{
|
||||
thrust::copy(d_range_srcs[sid], d_range_srcs[sid] + d_range_sizes[sid], d_range_dsts[sid]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class T>
|
||||
struct ge_t
|
||||
{
|
||||
T val;
|
||||
|
||||
__host__ __device__ bool operator()(T x)
|
||||
{
|
||||
return x >= val;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
std::size_t gen_uniform_offsets(
|
||||
executor exec,
|
||||
seed_t seed,
|
||||
cuda::std::span<T> segment_offsets,
|
||||
std::size_t min_segment_size,
|
||||
std::size_t max_segment_size)
|
||||
{
|
||||
const T total_elements = segment_offsets.size() - 2;
|
||||
|
||||
gen(exec,
|
||||
seed,
|
||||
segment_offsets,
|
||||
bit_entropy::_1_000,
|
||||
static_cast<T>(min_segment_size),
|
||||
static_cast<T>(max_segment_size));
|
||||
|
||||
auto tail = [&](const auto& policy) {
|
||||
thrust::fill_n(policy, segment_offsets.data() + total_elements, 1, total_elements + 1);
|
||||
thrust::exclusive_scan(
|
||||
policy, segment_offsets.data(), segment_offsets.data() + segment_offsets.size(), segment_offsets.data());
|
||||
auto iter = thrust::find_if(
|
||||
policy, segment_offsets.data(), segment_offsets.data() + segment_offsets.size(), ge_t<T>{total_elements});
|
||||
auto dist = cuda::std::distance(segment_offsets.data(), iter);
|
||||
thrust::fill_n(policy, segment_offsets.data() + dist, 1, total_elements);
|
||||
return dist + 1;
|
||||
};
|
||||
|
||||
if (exec == executor::device)
|
||||
{
|
||||
return tail(thrust::device);
|
||||
}
|
||||
|
||||
return tail(thrust::host);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Generates a vector of random key segments.
|
||||
*
|
||||
* Not all parameter combinations can be satisfied. For instance, if the total
|
||||
* elements is less than the minimal segment size, the function will return a
|
||||
* vector with a single element that is outside of the requested range.
|
||||
* At most one segment can be out of the requested range.
|
||||
*/
|
||||
template <typename T>
|
||||
void gen_uniform_key_segments_host(
|
||||
seed_t seed, cuda::std::span<T> keys, std::size_t min_segment_size, std::size_t max_segment_size)
|
||||
{
|
||||
thrust::host_vector<std::size_t> segment_offsets(keys.size() + 2);
|
||||
|
||||
{
|
||||
cuda::std::span<std::size_t> segment_offsets_span(
|
||||
thrust::raw_pointer_cast(segment_offsets.data()), segment_offsets.size());
|
||||
const std::size_t offsets_size =
|
||||
gen_uniform_offsets(executor::host, seed, segment_offsets_span, min_segment_size, max_segment_size);
|
||||
segment_offsets.resize(offsets_size);
|
||||
}
|
||||
|
||||
cuda::std::span<std::size_t> segment_offsets_span(
|
||||
thrust::raw_pointer_cast(segment_offsets.data()), segment_offsets.size());
|
||||
|
||||
gen_key_segments(executor::host, seed, keys, segment_offsets_span);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void gen_uniform_key_segments_device(
|
||||
seed_t seed, cuda::std::span<T> keys, std::size_t min_segment_size, std::size_t max_segment_size)
|
||||
{
|
||||
thrust::device_vector<std::size_t> segment_offsets(keys.size() + 2);
|
||||
|
||||
{
|
||||
cuda::std::span<std::size_t> segment_offsets_span(
|
||||
thrust::raw_pointer_cast(segment_offsets.data()), segment_offsets.size());
|
||||
const std::size_t offsets_size =
|
||||
gen_uniform_offsets(executor::device, seed, segment_offsets_span, min_segment_size, max_segment_size);
|
||||
segment_offsets.resize(offsets_size);
|
||||
}
|
||||
|
||||
cuda::std::span<std::size_t> segment_offsets_span(
|
||||
thrust::raw_pointer_cast(segment_offsets.data()), segment_offsets.size());
|
||||
|
||||
gen_key_segments(executor::device, seed, keys, segment_offsets_span);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::size_t gen_uniform_segment_offsets_host(
|
||||
seed_t seed, cuda::std::span<T> segment_offsets, std::size_t min_segment_size, std::size_t max_segment_size)
|
||||
{
|
||||
return gen_uniform_offsets(executor::host, seed, segment_offsets, min_segment_size, max_segment_size);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::size_t gen_uniform_segment_offsets_device(
|
||||
seed_t seed, cuda::std::span<T> segment_offsets, std::size_t min_segment_size, std::size_t max_segment_size)
|
||||
{
|
||||
return gen_uniform_offsets(executor::device, seed, segment_offsets, min_segment_size, max_segment_size);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void gen_power_law_segment_offsets_host(seed_t seed, cuda::std::span<T> segment_offsets, std::size_t elements)
|
||||
{
|
||||
generator_t{}.power_law_segment_offsets<T>(executor::host, seed, segment_offsets, elements);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void gen_power_law_segment_offsets_device(seed_t seed, cuda::std::span<T> segment_offsets, std::size_t elements)
|
||||
{
|
||||
generator_t{}.power_law_segment_offsets<T>(executor::device, seed, segment_offsets, elements);
|
||||
}
|
||||
|
||||
void do_not_optimize([[maybe_unused]] const void* ptr) {}
|
||||
} // namespace detail
|
||||
|
||||
#define INSTANTIATE(TYPE) \
|
||||
template void detail::gen_power_law_segment_offsets_host<TYPE>(seed_t, cuda::std::span<TYPE>, std::size_t); \
|
||||
template void detail::gen_power_law_segment_offsets_device<TYPE>(seed_t, cuda::std::span<TYPE>, std::size_t); \
|
||||
template std::size_t detail::gen_uniform_segment_offsets_host<TYPE>( \
|
||||
seed_t, cuda::std::span<TYPE>, std::size_t, std::size_t); \
|
||||
template std::size_t detail::gen_uniform_segment_offsets_device<TYPE>( \
|
||||
seed_t, cuda::std::span<TYPE>, std::size_t, std::size_t)
|
||||
|
||||
INSTANTIATE(int32_t);
|
||||
INSTANTIATE(uint32_t);
|
||||
INSTANTIATE(int64_t);
|
||||
INSTANTIATE(uint64_t);
|
||||
|
||||
#undef INSTANTIATE
|
||||
|
||||
// Instantiates only the uniform data generators used by non-segmented benchmarks (e.g. Reduce/Scan/RadixSort).
|
||||
#define INSTANTIATE_GEN(TYPE) \
|
||||
template void detail::gen_device<TYPE>(seed_t, cuda::std::span<TYPE>, bit_entropy, TYPE min, TYPE max); \
|
||||
template void detail::gen_host<TYPE>(seed_t, cuda::std::span<TYPE>, bit_entropy, TYPE min, TYPE max)
|
||||
|
||||
#define INSTANTIATE(TYPE) \
|
||||
template void detail::gen_uniform_key_segments_host<TYPE>(seed_t, cuda::std::span<TYPE>, std::size_t, std::size_t); \
|
||||
template void detail::gen_uniform_key_segments_device<TYPE>(seed_t, cuda::std::span<TYPE>, std::size_t, std::size_t); \
|
||||
INSTANTIATE_GEN(TYPE)
|
||||
|
||||
INSTANTIATE(bool);
|
||||
|
||||
INSTANTIATE(uint8_t);
|
||||
INSTANTIATE(uint16_t);
|
||||
INSTANTIATE(uint32_t);
|
||||
INSTANTIATE(uint64_t);
|
||||
|
||||
INSTANTIATE(int8_t);
|
||||
INSTANTIATE(int16_t);
|
||||
INSTANTIATE(int32_t);
|
||||
INSTANTIATE(int64_t);
|
||||
|
||||
#if _CCCL_HAS_INT128()
|
||||
INSTANTIATE(int128_t);
|
||||
INSTANTIATE(uint128_t);
|
||||
#endif
|
||||
|
||||
INSTANTIATE(float);
|
||||
INSTANTIATE(double);
|
||||
INSTANTIATE(complex32);
|
||||
INSTANTIATE(complex64);
|
||||
|
||||
// Extended floating-point types: only the uniform generators are needed (no segmented-sort key generators yet).
|
||||
#if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
|
||||
INSTANTIATE_GEN(__half);
|
||||
#endif
|
||||
#if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
|
||||
INSTANTIATE_GEN(__nv_bfloat16);
|
||||
#endif
|
||||
|
||||
#undef INSTANTIATE
|
||||
#undef INSTANTIATE_GEN
|
||||
777
cccl_upstream/nvbench_helper/nvbench_helper/nvbench_helper.cuh
Normal file
777
cccl_upstream/nvbench_helper/nvbench_helper/nvbench_helper.cuh
Normal file
@@ -0,0 +1,777 @@
|
||||
#pragma once
|
||||
|
||||
#include <cub/thread/thread_operators.cuh>
|
||||
|
||||
#include <thrust/device_vector.h>
|
||||
#include <thrust/execution_policy.h>
|
||||
|
||||
#include <cuda/std/cmath>
|
||||
#include <cuda/std/complex>
|
||||
#include <cuda/std/functional>
|
||||
#include <cuda/std/limits>
|
||||
#include <cuda/std/span>
|
||||
#include <cuda/std/type_traits>
|
||||
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
# include <cuda/memory_resource>
|
||||
# include <cuda/std/execution>
|
||||
# include <cuda/stream>
|
||||
#endif // THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
|
||||
#include <map>
|
||||
#include <stdexcept>
|
||||
|
||||
#include <nvbench/nvbench.cuh>
|
||||
|
||||
#if _CCCL_HAS_INT128()
|
||||
using int128_t = __int128_t;
|
||||
using uint128_t = __uint128_t;
|
||||
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(int128_t, "I128", "int128_t");
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(uint128_t, "U128", "uint128_t");
|
||||
#endif
|
||||
|
||||
using complex32 = cuda::std::complex<float>;
|
||||
using complex64 = cuda::std::complex<double>;
|
||||
|
||||
#if _CCCL_HAS_NVFP16()
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(__half, "F16", "half");
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(cuda::std::complex<__half>, "C16", "complex_half");
|
||||
#endif
|
||||
#if _CCCL_HAS_NVBF16()
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(__nv_bfloat16, "BF16", "bfloat16");
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(cuda::std::complex<__nv_bfloat16>, "CB16", "complex_bfloat16");
|
||||
#endif
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(complex32, "C32", "complex32");
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(complex64, "C64", "complex64");
|
||||
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(::cuda::std::false_type, "false", "false_type");
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(::cuda::std::true_type, "true", "true_type");
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(cub::detail::arg_min, "arg_min", "cub::detail::arg_min");
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(cub::detail::arg_max, "arg_max", "cub::detail::arg_max");
|
||||
|
||||
template <typename T, T I>
|
||||
struct nvbench::type_strings<::cuda::std::integral_constant<T, I>>
|
||||
{
|
||||
static std::string input_string()
|
||||
{
|
||||
return std::to_string(I);
|
||||
}
|
||||
static std::string description()
|
||||
{
|
||||
return "integral_constant<" + type_strings<T>::description() + ", " + std::to_string(I) + ">";
|
||||
}
|
||||
};
|
||||
|
||||
namespace detail
|
||||
{
|
||||
template <class List, class... Ts>
|
||||
struct push_back
|
||||
{};
|
||||
|
||||
template <class... As, class... Ts>
|
||||
struct push_back<nvbench::type_list<As...>, Ts...>
|
||||
{
|
||||
using type = nvbench::type_list<As..., Ts...>;
|
||||
};
|
||||
} // namespace detail
|
||||
|
||||
template <class List, class... Ts>
|
||||
using push_back_t = typename detail::push_back<List, Ts...>::type;
|
||||
|
||||
#ifdef TUNE_OffsetT
|
||||
using offset_types = nvbench::type_list<TUNE_OffsetT>;
|
||||
#else
|
||||
using offset_types = nvbench::type_list<int32_t, int64_t>;
|
||||
#endif
|
||||
|
||||
#ifdef TUNE_T
|
||||
using integral_types = nvbench::type_list<TUNE_T>;
|
||||
using fundamental_types = nvbench::type_list<TUNE_T>;
|
||||
using all_types = nvbench::type_list<TUNE_T>;
|
||||
#else
|
||||
// keep those lists in sync with the documentation in tuning_infra.rst
|
||||
using integral_types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t>;
|
||||
|
||||
using fundamental_types =
|
||||
nvbench::type_list<int8_t,
|
||||
int16_t,
|
||||
int32_t,
|
||||
int64_t,
|
||||
# if _CCCL_HAS_INT128()
|
||||
int128_t,
|
||||
# endif
|
||||
float,
|
||||
double>;
|
||||
|
||||
using all_types =
|
||||
nvbench::type_list<int8_t,
|
||||
int16_t,
|
||||
int32_t,
|
||||
int64_t,
|
||||
# if _CCCL_HAS_INT128()
|
||||
int128_t,
|
||||
# endif
|
||||
float,
|
||||
double,
|
||||
complex32>;
|
||||
#endif
|
||||
|
||||
template <class T>
|
||||
class value_wrapper_t
|
||||
{
|
||||
T m_val{};
|
||||
|
||||
public:
|
||||
explicit value_wrapper_t(T val)
|
||||
: m_val(val)
|
||||
{}
|
||||
|
||||
T get() const
|
||||
{
|
||||
return m_val;
|
||||
}
|
||||
|
||||
value_wrapper_t& operator++()
|
||||
{
|
||||
m_val++;
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
||||
class seed_t : public value_wrapper_t<unsigned long long int>
|
||||
{
|
||||
public:
|
||||
using value_wrapper_t::value_wrapper_t;
|
||||
using value_wrapper_t::operator++;
|
||||
|
||||
seed_t()
|
||||
: value_wrapper_t(42)
|
||||
{}
|
||||
};
|
||||
|
||||
enum class bit_entropy
|
||||
{
|
||||
_1_000 = 0,
|
||||
_0_811 = 1,
|
||||
_0_544 = 2,
|
||||
_0_337 = 3,
|
||||
_0_201 = 4,
|
||||
_0_000 = 4200
|
||||
};
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(bit_entropy, "BE", "bit entropy");
|
||||
|
||||
[[nodiscard]] inline double entropy_to_probability(bit_entropy entropy)
|
||||
{
|
||||
switch (entropy)
|
||||
{
|
||||
case bit_entropy::_1_000:
|
||||
return 1.0;
|
||||
case bit_entropy::_0_811:
|
||||
return 0.811;
|
||||
case bit_entropy::_0_544:
|
||||
return 0.544;
|
||||
case bit_entropy::_0_337:
|
||||
return 0.337;
|
||||
case bit_entropy::_0_201:
|
||||
return 0.201;
|
||||
case bit_entropy::_0_000:
|
||||
[[fallthrough]];
|
||||
default:
|
||||
return 0.0;
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] inline bit_entropy str_to_entropy(std::string str)
|
||||
{
|
||||
if (str == "1.000")
|
||||
{
|
||||
return bit_entropy::_1_000;
|
||||
}
|
||||
else if (str == "0.811")
|
||||
{
|
||||
return bit_entropy::_0_811;
|
||||
}
|
||||
else if (str == "0.544")
|
||||
{
|
||||
return bit_entropy::_0_544;
|
||||
}
|
||||
else if (str == "0.337")
|
||||
{
|
||||
return bit_entropy::_0_337;
|
||||
}
|
||||
else if (str == "0.201")
|
||||
{
|
||||
return bit_entropy::_0_201;
|
||||
}
|
||||
else if (str == "0.000")
|
||||
{
|
||||
return bit_entropy::_0_000;
|
||||
}
|
||||
|
||||
throw std::runtime_error("Can't convert string to bit entropy");
|
||||
}
|
||||
|
||||
// Creates an interpolated value of type T between min (at = 0.0) and max (at = 1.0).
|
||||
template <typename T>
|
||||
[[nodiscard]] T lerp_min_max(double at) noexcept
|
||||
{
|
||||
if (at == 1.0)
|
||||
{
|
||||
return ::cuda::std::numeric_limits<T>::max();
|
||||
}
|
||||
const auto min_val = static_cast<double>(::cuda::std::numeric_limits<T>::lowest());
|
||||
const auto max_val = static_cast<double>(::cuda::std::numeric_limits<T>::max());
|
||||
return static_cast<T>(::cuda::std::lerp(min_val, max_val, at));
|
||||
}
|
||||
|
||||
namespace detail
|
||||
{
|
||||
void do_not_optimize(const void* ptr);
|
||||
|
||||
template <typename T>
|
||||
void gen_host(seed_t seed, cuda::std::span<T> data, bit_entropy entropy, T min, T max);
|
||||
|
||||
template <typename T>
|
||||
void gen_device(seed_t seed, cuda::std::span<T> data, bit_entropy entropy, T min, T max);
|
||||
|
||||
template <typename T>
|
||||
void gen_uniform_key_segments_host(
|
||||
seed_t seed, cuda::std::span<T> data, std::size_t min_segment_size, std::size_t max_segment_size);
|
||||
|
||||
template <typename T>
|
||||
void gen_uniform_key_segments_device(
|
||||
seed_t seed, cuda::std::span<T> data, std::size_t min_segment_size, std::size_t max_segment_size);
|
||||
|
||||
template <typename T>
|
||||
std::size_t gen_uniform_segment_offsets_host(
|
||||
seed_t seed, cuda::std::span<T> segment_offsets, std::size_t min_segment_size, std::size_t max_segment_size);
|
||||
|
||||
template <typename T>
|
||||
std::size_t gen_uniform_segment_offsets_device(
|
||||
seed_t seed, cuda::std::span<T> segment_offsets, std::size_t min_segment_size, std::size_t max_segment_size);
|
||||
|
||||
template <typename T>
|
||||
void gen_power_law_segment_offsets_host(seed_t seed, cuda::std::span<T> segment_offsets, std::size_t elements);
|
||||
|
||||
template <typename T>
|
||||
void gen_power_law_segment_offsets_device(seed_t seed, cuda::std::span<T> segment_offsets, std::size_t elements);
|
||||
|
||||
namespace
|
||||
{
|
||||
struct generator_base_t
|
||||
{
|
||||
seed_t m_seed{};
|
||||
const std::size_t m_elements{0};
|
||||
const bit_entropy m_entropy{bit_entropy::_1_000};
|
||||
|
||||
template <typename T>
|
||||
thrust::device_vector<T> generate(T min, T max)
|
||||
{
|
||||
thrust::device_vector<T> vec(m_elements);
|
||||
cuda::std::span<T> span(thrust::raw_pointer_cast(vec.data()), m_elements);
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
gen_device(m_seed, span, m_entropy, min, max);
|
||||
#else
|
||||
gen_host(m_seed, span, m_entropy, min, max);
|
||||
#endif
|
||||
++m_seed;
|
||||
return vec;
|
||||
}
|
||||
};
|
||||
|
||||
template <class T>
|
||||
struct vector_generator_t : generator_base_t
|
||||
{
|
||||
const T m_min{::cuda::std::numeric_limits<T>::min()};
|
||||
const T m_max{::cuda::std::numeric_limits<T>::max()};
|
||||
|
||||
operator thrust::device_vector<T>()
|
||||
{
|
||||
return generator_base_t::generate(m_min, m_max);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct vector_generator_t<void> : generator_base_t
|
||||
{
|
||||
template <typename T>
|
||||
operator thrust::device_vector<T>()
|
||||
{
|
||||
return generator_base_t::generate(::cuda::std::numeric_limits<T>::min(), ::cuda::std::numeric_limits<T>::max());
|
||||
}
|
||||
|
||||
// This overload is needed because numeric limits is not specialized for complex, making
|
||||
// the min and max values for complex equal zero.
|
||||
template <typename T>
|
||||
operator thrust::device_vector<::cuda::std::complex<T>>()
|
||||
{
|
||||
const auto min =
|
||||
::cuda::std::complex<T>{::cuda::std::numeric_limits<T>::min(), ::cuda::std::numeric_limits<T>::min()};
|
||||
const auto max =
|
||||
::cuda::std::complex<T>{::cuda::std::numeric_limits<T>::max(), ::cuda::std::numeric_limits<T>::max()};
|
||||
|
||||
return generator_base_t::generate(min, max);
|
||||
}
|
||||
};
|
||||
|
||||
struct uniform_key_segments_generator_t
|
||||
{
|
||||
seed_t m_seed{};
|
||||
const std::size_t m_total_elements{0};
|
||||
const std::size_t m_min_segment_size{0};
|
||||
const std::size_t m_max_segment_size{0};
|
||||
|
||||
template <class KeyT>
|
||||
operator thrust::device_vector<KeyT>()
|
||||
{
|
||||
thrust::device_vector<KeyT> keys_vec(m_total_elements);
|
||||
cuda::std::span<KeyT> keys(thrust::raw_pointer_cast(keys_vec.data()), keys_vec.size());
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
gen_uniform_key_segments_device(m_seed, keys, m_min_segment_size, m_max_segment_size);
|
||||
#else
|
||||
gen_uniform_key_segments_host(m_seed, keys, m_min_segment_size, m_max_segment_size);
|
||||
#endif
|
||||
++m_seed;
|
||||
return keys_vec;
|
||||
}
|
||||
};
|
||||
|
||||
struct uniform_segment_offsets_generator_t
|
||||
{
|
||||
seed_t m_seed{};
|
||||
const std::size_t m_total_elements{0};
|
||||
const std::size_t m_min_segment_size{0};
|
||||
const std::size_t m_max_segment_size{0};
|
||||
|
||||
template <class OffsetT>
|
||||
operator thrust::device_vector<OffsetT>()
|
||||
{
|
||||
thrust::device_vector<OffsetT> offsets_vec(m_total_elements + 2);
|
||||
cuda::std::span<OffsetT> offsets(thrust::raw_pointer_cast(offsets_vec.data()), offsets_vec.size());
|
||||
const std::size_t offsets_size =
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
gen_uniform_segment_offsets_device(m_seed, offsets, m_min_segment_size, m_max_segment_size);
|
||||
#else
|
||||
gen_uniform_segment_offsets_host(m_seed, offsets, m_min_segment_size, m_max_segment_size);
|
||||
#endif
|
||||
offsets_vec.resize(offsets_size);
|
||||
offsets_vec.shrink_to_fit();
|
||||
++m_seed;
|
||||
return offsets_vec;
|
||||
}
|
||||
};
|
||||
|
||||
struct power_law_segment_offsets_generator_t
|
||||
{
|
||||
seed_t m_seed{};
|
||||
const std::size_t m_elements{0};
|
||||
const std::size_t m_segments{0};
|
||||
|
||||
template <class OffsetT>
|
||||
operator thrust::device_vector<OffsetT>()
|
||||
{
|
||||
thrust::device_vector<OffsetT> offsets_vec(m_segments + 1);
|
||||
cuda::std::span<OffsetT> offsets(thrust::raw_pointer_cast(offsets_vec.data()), offsets_vec.size());
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
gen_power_law_segment_offsets_device(m_seed, offsets, m_elements);
|
||||
#else
|
||||
gen_power_law_segment_offsets_host(m_seed, offsets, m_elements);
|
||||
#endif
|
||||
++m_seed;
|
||||
return offsets_vec;
|
||||
}
|
||||
};
|
||||
|
||||
struct gen_uniform_key_segments_t
|
||||
{
|
||||
uniform_key_segments_generator_t
|
||||
operator()(std::size_t total_elements, std::size_t min_segment_size, std::size_t max_segment_size) const
|
||||
{
|
||||
return {seed_t{}, total_elements, min_segment_size, max_segment_size};
|
||||
}
|
||||
};
|
||||
|
||||
struct gen_uniform_segment_offsets_t
|
||||
{
|
||||
uniform_segment_offsets_generator_t
|
||||
operator()(std::size_t total_elements, std::size_t min_segment_size, std::size_t max_segment_size) const
|
||||
{
|
||||
return {seed_t{}, total_elements, min_segment_size, max_segment_size};
|
||||
}
|
||||
};
|
||||
|
||||
struct gen_uniform_t
|
||||
{
|
||||
gen_uniform_key_segments_t key_segments{};
|
||||
gen_uniform_segment_offsets_t segment_offsets{};
|
||||
};
|
||||
|
||||
struct gen_power_law_segment_offsets_t
|
||||
{
|
||||
power_law_segment_offsets_generator_t operator()(std::size_t elements, std::size_t segments) const
|
||||
{
|
||||
return {seed_t{}, elements, segments};
|
||||
}
|
||||
};
|
||||
|
||||
struct gen_power_law_t
|
||||
{
|
||||
gen_power_law_segment_offsets_t segment_offsets{};
|
||||
};
|
||||
|
||||
struct gen_t
|
||||
{
|
||||
vector_generator_t<void> operator()(std::size_t elements, bit_entropy entropy = bit_entropy::_1_000) const
|
||||
{
|
||||
return {seed_t{}, elements, entropy};
|
||||
}
|
||||
|
||||
template <class T>
|
||||
vector_generator_t<T> operator()(
|
||||
std::size_t elements,
|
||||
bit_entropy entropy = bit_entropy::_1_000,
|
||||
T min = ::cuda::std::numeric_limits<T>::min,
|
||||
T max = ::cuda::std::numeric_limits<T>::max()) const
|
||||
{
|
||||
return {{seed_t{}, elements, entropy}, min, max};
|
||||
}
|
||||
|
||||
gen_uniform_t uniform{};
|
||||
gen_power_law_t power_law{};
|
||||
};
|
||||
} // namespace
|
||||
} // namespace detail
|
||||
|
||||
inline detail::gen_t generate;
|
||||
|
||||
template <class T>
|
||||
void do_not_optimize(const T& val)
|
||||
{
|
||||
detail::do_not_optimize(&val);
|
||||
}
|
||||
|
||||
struct less_t
|
||||
{
|
||||
template <typename DataType>
|
||||
__host__ __device__ bool operator()(const DataType& lhs, const DataType& rhs) const
|
||||
{
|
||||
return lhs < rhs;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ __device__ inline bool
|
||||
operator()(const ::cuda::std::complex<T>& lhs, const ::cuda::std::complex<T>& rhs) const
|
||||
{
|
||||
double magnitude_0 = cuda::std::abs(lhs);
|
||||
double magnitude_1 = cuda::std::abs(rhs);
|
||||
|
||||
if (cuda::std::isnan(magnitude_0) || cuda::std::isnan(magnitude_1))
|
||||
{
|
||||
// NaN's are always equal.
|
||||
return false;
|
||||
}
|
||||
else if (cuda::std::isinf(magnitude_0) || cuda::std::isinf(magnitude_1))
|
||||
{
|
||||
// If the real or imaginary part of the complex number has a very large value
|
||||
// (close to the maximum representable value for a double), it is possible that
|
||||
// the magnitude computation can result in positive infinity:
|
||||
// ```cpp
|
||||
// const double large_number = ::cuda::std::numeric_limits<double>::max() / 2;
|
||||
// std::complex<double> z(large_number, large_number);
|
||||
// std::abs(z) == inf;
|
||||
// ```
|
||||
// Dividing both components by a constant before computing the magnitude prevents overflow.
|
||||
const T scaler = 0.5;
|
||||
|
||||
magnitude_0 = cuda::std::abs(lhs * scaler);
|
||||
magnitude_1 = cuda::std::abs(rhs * scaler);
|
||||
}
|
||||
|
||||
const T difference = cuda::std::abs(magnitude_0 - magnitude_1);
|
||||
const T threshold = ::cuda::std::numeric_limits<T>::epsilon() * 2;
|
||||
|
||||
if (difference < threshold)
|
||||
{
|
||||
// Triangles with the same magnitude are sorted by their phase angle.
|
||||
const T phase_angle_0 = cuda::std::arg(lhs);
|
||||
const T phase_angle_1 = cuda::std::arg(rhs);
|
||||
|
||||
return phase_angle_0 < phase_angle_1;
|
||||
}
|
||||
else
|
||||
{
|
||||
return magnitude_0 < magnitude_1;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
struct max_t
|
||||
{
|
||||
template <typename DataType>
|
||||
__host__ __device__ DataType operator()(const DataType& lhs, const DataType& rhs) const
|
||||
{
|
||||
less_t less{};
|
||||
return less(lhs, rhs) ? rhs : lhs;
|
||||
}
|
||||
|
||||
#if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
|
||||
__host__ __device__ __half operator()(__half lhs, __half rhs) const
|
||||
{
|
||||
return static_cast<float>(lhs) < static_cast<float>(rhs) ? rhs : lhs;
|
||||
}
|
||||
#endif // _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
|
||||
|
||||
#if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
|
||||
__host__ __device__ __nv_bfloat16 operator()(__nv_bfloat16 lhs, __nv_bfloat16 rhs) const
|
||||
{
|
||||
return static_cast<float>(lhs) < static_cast<float>(rhs) ? rhs : lhs;
|
||||
}
|
||||
#endif // _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
|
||||
};
|
||||
|
||||
template <class T>
|
||||
struct less_then_t
|
||||
{
|
||||
T m_val;
|
||||
|
||||
[[nodiscard]] __device__ bool operator()(const T& val) const noexcept
|
||||
{
|
||||
return val < m_val;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA
|
||||
template <typename T>
|
||||
struct proclaims_copyable_arguments<less_then_t<T>> : ::cuda::std::true_type
|
||||
{};
|
||||
_CCCL_END_NAMESPACE_CUDA
|
||||
|
||||
namespace
|
||||
{
|
||||
struct caching_allocator_t
|
||||
{
|
||||
using value_type = char;
|
||||
|
||||
caching_allocator_t() = default;
|
||||
~caching_allocator_t()
|
||||
{
|
||||
free_all();
|
||||
}
|
||||
|
||||
char* allocate(std::ptrdiff_t num_bytes)
|
||||
{
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
if (first_async_stream != ::cuda::invalid_stream)
|
||||
{
|
||||
// there was already an async allocate
|
||||
throw std::runtime_error("caching_allocator_t is not intended to be used asynchronously and synchronously at the "
|
||||
"same time");
|
||||
}
|
||||
#endif // THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
|
||||
value_type* result{};
|
||||
auto free_block = free_blocks.find(num_bytes);
|
||||
|
||||
if (free_block != free_blocks.end())
|
||||
{
|
||||
result = free_block->second;
|
||||
free_blocks.erase(free_block);
|
||||
}
|
||||
else
|
||||
{
|
||||
result = do_allocate(num_bytes);
|
||||
}
|
||||
|
||||
allocated_blocks.insert(std::make_pair(result, num_bytes));
|
||||
return result;
|
||||
}
|
||||
|
||||
void deallocate(char* ptr, size_t, [[maybe_unused]] bool check_stream = true)
|
||||
{
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
if (check_stream && first_async_stream != ::cuda::invalid_stream)
|
||||
{
|
||||
// there was already an async allocate
|
||||
throw std::runtime_error("caching_allocator_t is not intended to be used asynchronously and synchronously at the "
|
||||
"same time");
|
||||
}
|
||||
#endif // THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
|
||||
auto iter = allocated_blocks.find(ptr);
|
||||
if (iter == allocated_blocks.end())
|
||||
{
|
||||
throw std::runtime_error("Memory was not allocated by this allocator");
|
||||
}
|
||||
|
||||
std::ptrdiff_t num_bytes = iter->second;
|
||||
allocated_blocks.erase(iter);
|
||||
free_blocks.insert(std::make_pair(num_bytes, ptr));
|
||||
}
|
||||
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
void* allocate_sync(size_t num_bytes, size_t)
|
||||
{
|
||||
return allocate(static_cast<std::ptrdiff_t>(num_bytes));
|
||||
}
|
||||
|
||||
void deallocate_sync(void* ptr, size_t num_bytes, size_t)
|
||||
{
|
||||
deallocate(static_cast<char*>(ptr), num_bytes);
|
||||
}
|
||||
|
||||
void* allocate(::cuda::stream_ref __stream, size_t num_bytes, size_t)
|
||||
{
|
||||
if (first_async_stream == ::cuda::invalid_stream)
|
||||
{
|
||||
first_async_stream = __stream;
|
||||
}
|
||||
else if (first_async_stream != __stream)
|
||||
{
|
||||
throw std::runtime_error("caching_allocator_t is not intended to be used asynchronously from multiple streams");
|
||||
}
|
||||
|
||||
value_type* result{};
|
||||
auto free_block = free_blocks.find(static_cast<std::ptrdiff_t>(num_bytes));
|
||||
|
||||
if (free_block != free_blocks.end())
|
||||
{
|
||||
result = free_block->second;
|
||||
free_blocks.erase(free_block);
|
||||
}
|
||||
else
|
||||
{
|
||||
const cudaError_t status = cudaMallocAsync(&result, num_bytes, __stream.get());
|
||||
if (cudaSuccess != status)
|
||||
{
|
||||
throw std::runtime_error(std::string("Failed to allocate device memory: ") + cudaGetErrorString(status));
|
||||
}
|
||||
// NOTE: We can avoid a `__stream.sync()` here because `allocate` is only ever called asynchronously with a stream
|
||||
// So it is fine that the memory is only valid in stream order
|
||||
}
|
||||
|
||||
allocated_blocks.insert(std::make_pair(result, num_bytes));
|
||||
return result;
|
||||
}
|
||||
|
||||
void deallocate(::cuda::stream_ref __stream, void* ptr, size_t num_bytes, size_t)
|
||||
{
|
||||
if (first_async_stream != __stream)
|
||||
{
|
||||
throw std::runtime_error("caching_allocator_t is not intended to be used asynchronously from multiple streams");
|
||||
}
|
||||
|
||||
// there is no need to sync the stream here and we can just insert the allocation into the free list, because the
|
||||
// next allocation can only be done from the same stream again.
|
||||
deallocate(static_cast<char*>(ptr), num_bytes, false);
|
||||
}
|
||||
#endif // THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
|
||||
private:
|
||||
using free_blocks_type = std::multimap<std::ptrdiff_t, char*>;
|
||||
using allocated_blocks_type = std::map<char*, std::ptrdiff_t>;
|
||||
|
||||
free_blocks_type free_blocks;
|
||||
allocated_blocks_type allocated_blocks;
|
||||
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
::cuda::stream_ref first_async_stream{::cuda::invalid_stream}; // just to detect wrong usage patterns
|
||||
#endif // THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
|
||||
void free_all()
|
||||
{
|
||||
for (auto i : free_blocks)
|
||||
{
|
||||
do_deallocate(i.second);
|
||||
}
|
||||
|
||||
for (auto i : allocated_blocks)
|
||||
{
|
||||
do_deallocate(i.first);
|
||||
}
|
||||
}
|
||||
|
||||
value_type* do_allocate(std::size_t num_bytes)
|
||||
{
|
||||
value_type* result{};
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
const cudaError_t status = cudaMalloc(&result, num_bytes);
|
||||
if (cudaSuccess != status)
|
||||
{
|
||||
throw std::runtime_error(std::string("Failed to allocate device memory: ") + cudaGetErrorString(status));
|
||||
}
|
||||
#else
|
||||
result = new value_type[num_bytes];
|
||||
#endif
|
||||
return result;
|
||||
}
|
||||
|
||||
void do_deallocate(value_type* ptr)
|
||||
{
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
cudaFree(ptr);
|
||||
#else
|
||||
delete[] ptr;
|
||||
#endif
|
||||
}
|
||||
|
||||
friend bool operator==(const caching_allocator_t& lhs, const caching_allocator_t& rhs)
|
||||
{
|
||||
return lhs.free_blocks == rhs.free_blocks && lhs.allocated_blocks == rhs.allocated_blocks;
|
||||
}
|
||||
|
||||
friend bool operator!=(const caching_allocator_t& lhs, const caching_allocator_t& rhs)
|
||||
{
|
||||
return lhs.free_blocks == rhs.free_blocks && lhs.allocated_blocks == rhs.allocated_blocks;
|
||||
}
|
||||
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
friend constexpr void get_property(const caching_allocator_t&, cuda::mr::device_accessible) noexcept {}
|
||||
#endif // THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
};
|
||||
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
auto policy(caching_allocator_t& alloc)
|
||||
{
|
||||
return thrust::cuda::par(alloc);
|
||||
}
|
||||
auto cuda_policy(caching_allocator_t& alloc)
|
||||
{
|
||||
return cuda::execution::gpu.with(cuda::mr::get_memory_resource, alloc);
|
||||
}
|
||||
#else
|
||||
auto policy(caching_allocator_t&)
|
||||
{
|
||||
return thrust::device;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
auto policy(caching_allocator_t& alloc, nvbench::launch& launch)
|
||||
{
|
||||
return thrust::cuda::par(alloc).on(launch.get_stream());
|
||||
}
|
||||
auto cuda_policy(caching_allocator_t& alloc, nvbench::launch& launch)
|
||||
{
|
||||
return cuda::execution::gpu.with(cuda::mr::get_memory_resource, alloc)
|
||||
.with(cuda::get_stream, launch.get_stream().get_stream());
|
||||
}
|
||||
#else
|
||||
auto policy(caching_allocator_t&, nvbench::launch&)
|
||||
{
|
||||
return thrust::device;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
// Returns an environment for benchmarking using alloc as MR, launch's stream, and any additional envs passed in.
|
||||
template <typename... MoreEnvs>
|
||||
auto cub_bench_env(caching_allocator_t& alloc, nvbench::launch& launch, MoreEnvs... envs)
|
||||
{
|
||||
return cuda::std::execution::env{
|
||||
::cuda::stream_ref{launch.get_stream().get_stream()},
|
||||
::cuda::std::execution::prop{cuda::mr::get_memory_resource, ::cuda::mr::resource_ref<>{alloc}},
|
||||
envs...};
|
||||
}
|
||||
#endif // THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
} // namespace
|
||||
157
cccl_upstream/nvbench_helper/test/gen_entropy.cu
Normal file
157
cccl_upstream/nvbench_helper/test/gen_entropy.cu
Normal file
@@ -0,0 +1,157 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
|
||||
// SPDX-License-Identifier: BSD-3
|
||||
|
||||
#include <cub/device/device_run_length_encode.cuh>
|
||||
|
||||
#include <thrust/count.h>
|
||||
#include <thrust/execution_policy.h>
|
||||
#include <thrust/host_vector.h>
|
||||
#include <thrust/sort.h>
|
||||
#include <thrust/transform.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
|
||||
#include <nvbench_helper.cuh>
|
||||
|
||||
#include <catch2/catch_template_test_macros.hpp>
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
|
||||
template <class T>
|
||||
double get_expected_entropy(bit_entropy in_entropy)
|
||||
{
|
||||
if (in_entropy == bit_entropy::_0_000)
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
if (in_entropy == bit_entropy::_1_000)
|
||||
{
|
||||
return sizeof(T) * 8;
|
||||
}
|
||||
|
||||
const int samples = static_cast<int>(in_entropy) + 1;
|
||||
const double p1 = std::pow(0.5, samples);
|
||||
const double p2 = 1 - p1;
|
||||
const double entropy = (-p1 * std::log2(p1)) + (-p2 * std::log2(p2));
|
||||
return sizeof(T) * 8 * entropy;
|
||||
}
|
||||
|
||||
template <class T>
|
||||
double compute_actual_entropy(thrust::device_vector<T> in)
|
||||
{
|
||||
const int n = static_cast<int>(in.size());
|
||||
|
||||
#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA
|
||||
thrust::device_vector<T> unique(n);
|
||||
thrust::device_vector<int> counts(n);
|
||||
thrust::device_vector<int> num_runs(1);
|
||||
thrust::sort(in.begin(), in.end(), less_t{});
|
||||
|
||||
// RLE
|
||||
void* d_temp_storage = nullptr;
|
||||
std::size_t temp_storage_bytes = 0;
|
||||
|
||||
T* d_in = thrust::raw_pointer_cast(in.data());
|
||||
T* d_unique_out = thrust::raw_pointer_cast(unique.data());
|
||||
int* d_counts_out = thrust::raw_pointer_cast(counts.data());
|
||||
int* d_num_runs_out = thrust::raw_pointer_cast(num_runs.data());
|
||||
|
||||
cub::DeviceRunLengthEncode::Encode(
|
||||
d_temp_storage, temp_storage_bytes, d_in, d_unique_out, d_counts_out, d_num_runs_out, n);
|
||||
|
||||
thrust::device_vector<std::uint8_t> temp_storage(temp_storage_bytes);
|
||||
d_temp_storage = thrust::raw_pointer_cast(temp_storage.data());
|
||||
|
||||
cub::DeviceRunLengthEncode::Encode(
|
||||
d_temp_storage, temp_storage_bytes, d_in, d_unique_out, d_counts_out, d_num_runs_out, n);
|
||||
|
||||
thrust::host_vector<int> h_counts = counts;
|
||||
thrust::host_vector<int> h_num_runs = num_runs;
|
||||
#else
|
||||
std::vector<T> h_in(in.begin(), in.end());
|
||||
std::sort(h_in.begin(), h_in.end(), less_t{});
|
||||
thrust::host_vector<int> h_counts;
|
||||
T prev = h_in[0];
|
||||
int length = 1;
|
||||
|
||||
for (std::size_t i = 1; i < h_in.size(); i++)
|
||||
{
|
||||
const T next = h_in[i];
|
||||
if (next == prev)
|
||||
{
|
||||
length++;
|
||||
}
|
||||
else
|
||||
{
|
||||
h_counts.push_back(length);
|
||||
prev = next;
|
||||
length = 1;
|
||||
}
|
||||
}
|
||||
h_counts.push_back(length);
|
||||
|
||||
thrust::host_vector<int> h_num_runs(1, h_counts.size());
|
||||
#endif
|
||||
|
||||
// normalize counts
|
||||
thrust::host_vector<double> ps(h_num_runs[0]);
|
||||
for (std::size_t i = 0; i < ps.size(); i++)
|
||||
{
|
||||
ps[i] = static_cast<double>(h_counts[i]) / n;
|
||||
}
|
||||
|
||||
double entropy = 0.0;
|
||||
|
||||
if (ps.size())
|
||||
{
|
||||
for (double p : ps)
|
||||
{
|
||||
entropy -= p * std::log2(p);
|
||||
}
|
||||
}
|
||||
|
||||
return entropy;
|
||||
}
|
||||
|
||||
TEMPLATE_LIST_TEST_CASE("Generators produce data with given entropy", "[gen]", fundamental_types)
|
||||
{
|
||||
constexpr int num_entropy_levels = 6;
|
||||
std::array<bit_entropy, num_entropy_levels> entropy_levels{
|
||||
bit_entropy::_0_000,
|
||||
bit_entropy::_0_201,
|
||||
bit_entropy::_0_337,
|
||||
bit_entropy::_0_544,
|
||||
bit_entropy::_0_811,
|
||||
bit_entropy::_1_000};
|
||||
|
||||
std::vector<double> entropy(num_entropy_levels);
|
||||
std::transform(entropy_levels.cbegin(), entropy_levels.cend(), entropy.begin(), [](bit_entropy entropy) {
|
||||
const thrust::device_vector<TestType> data = generate(1 << 24, entropy);
|
||||
return compute_actual_entropy(data);
|
||||
});
|
||||
|
||||
REQUIRE(std::is_sorted(entropy.begin(), entropy.end(), less_t{}));
|
||||
REQUIRE(std::unique(entropy.begin(), entropy.end()) == entropy.end());
|
||||
}
|
||||
|
||||
TEST_CASE("Generators support bool", "[gen]")
|
||||
{
|
||||
constexpr int num_entropy_levels = 6;
|
||||
std::array<bit_entropy, num_entropy_levels> entropy_levels{
|
||||
bit_entropy::_0_000,
|
||||
bit_entropy::_0_201,
|
||||
bit_entropy::_0_337,
|
||||
bit_entropy::_0_544,
|
||||
bit_entropy::_0_811,
|
||||
bit_entropy::_1_000};
|
||||
|
||||
std::vector<std::size_t> number_of_set(num_entropy_levels);
|
||||
std::transform(entropy_levels.cbegin(), entropy_levels.cend(), number_of_set.begin(), [](bit_entropy entropy) {
|
||||
const thrust::device_vector<bool> data = generate(1 << 24, entropy);
|
||||
return thrust::count(data.begin(), data.end(), true);
|
||||
});
|
||||
|
||||
REQUIRE(std::is_sorted(number_of_set.begin(), number_of_set.end()));
|
||||
REQUIRE(std::unique(number_of_set.begin(), number_of_set.end()) == number_of_set.end());
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
|
||||
// SPDX-License-Identifier: BSD-3
|
||||
|
||||
#include <thrust/device_vector.h>
|
||||
#include <thrust/host_vector.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
|
||||
#include <nvbench_helper.cuh>
|
||||
|
||||
#include <boost/math/statistics/anderson_darling.hpp>
|
||||
#include <boost/math/statistics/univariate_statistics.hpp>
|
||||
#include <catch2/catch_template_test_macros.hpp>
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
|
||||
bool is_normal(thrust::host_vector<double> data)
|
||||
{
|
||||
std::sort(data.begin(), data.end());
|
||||
const double A2 = boost::math::statistics::anderson_darling_normality_statistic(data);
|
||||
return A2 / data.size() < 0.05;
|
||||
}
|
||||
|
||||
using types = nvbench::type_list<uint32_t, uint64_t>;
|
||||
|
||||
TEMPLATE_LIST_TEST_CASE("Generators produce power law distributed data", "[gen][power-law]", types)
|
||||
{
|
||||
const std::size_t elements = 1 << 28;
|
||||
const std::size_t segments = 4 * 1024;
|
||||
const thrust::device_vector<TestType> d_segment_offsets = generate.power_law.segment_offsets(elements, segments);
|
||||
REQUIRE(d_segment_offsets.size() == segments + 1);
|
||||
|
||||
std::size_t actual_elements = 0;
|
||||
thrust::host_vector<double> log_sizes(segments);
|
||||
const thrust::host_vector<TestType> h_segment_offsets = d_segment_offsets;
|
||||
for (std::size_t i = 0; i < segments; ++i)
|
||||
{
|
||||
const TestType begin = h_segment_offsets[i];
|
||||
const TestType end = h_segment_offsets[i + 1];
|
||||
REQUIRE(begin <= end);
|
||||
|
||||
const std::size_t size = end - begin;
|
||||
actual_elements += size;
|
||||
log_sizes[i] = std::log(size);
|
||||
}
|
||||
|
||||
REQUIRE(actual_elements == elements);
|
||||
REQUIRE(is_normal(std::move(log_sizes)));
|
||||
}
|
||||
37
cccl_upstream/nvbench_helper/test/gen_range.cu
Normal file
37
cccl_upstream/nvbench_helper/test/gen_range.cu
Normal file
@@ -0,0 +1,37 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
|
||||
// SPDX-License-Identifier: BSD-3
|
||||
|
||||
#include <thrust/device_vector.h>
|
||||
#include <thrust/extrema.h>
|
||||
|
||||
#include <limits>
|
||||
|
||||
#include <nvbench_helper.cuh>
|
||||
|
||||
#include <catch2/catch_template_test_macros.hpp>
|
||||
#include <catch2/generators/catch_generators_all.hpp>
|
||||
|
||||
using types =
|
||||
nvbench::type_list<int8_t,
|
||||
int16_t,
|
||||
int32_t,
|
||||
int64_t,
|
||||
#if _CCCL_HAS_INT128()
|
||||
int128_t,
|
||||
#endif
|
||||
float,
|
||||
double>;
|
||||
|
||||
TEMPLATE_LIST_TEST_CASE("Generators produce data within specified range", "[gen]", types)
|
||||
{
|
||||
const auto min = static_cast<TestType>(GENERATE_COPY(take(3, random(-124, 0))));
|
||||
const auto max = static_cast<TestType>(GENERATE_COPY(take(3, random(0, 124))));
|
||||
|
||||
const thrust::device_vector<TestType> data = generate(1 << 16, bit_entropy::_1_000, min, max);
|
||||
|
||||
const TestType min_element = *thrust::min_element(data.begin(), data.end());
|
||||
const TestType max_element = *thrust::max_element(data.begin(), data.end());
|
||||
|
||||
REQUIRE(min_element >= min);
|
||||
REQUIRE(max_element <= max);
|
||||
}
|
||||
34
cccl_upstream/nvbench_helper/test/gen_seed.cu
Normal file
34
cccl_upstream/nvbench_helper/test/gen_seed.cu
Normal file
@@ -0,0 +1,34 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
|
||||
// SPDX-License-Identifier: BSD-3
|
||||
|
||||
#include <thrust/device_vector.h>
|
||||
#include <thrust/equal.h>
|
||||
|
||||
#include <nvbench_helper.cuh>
|
||||
|
||||
#include <catch2/catch_template_test_macros.hpp>
|
||||
|
||||
using types = nvbench::type_list<
|
||||
bool,
|
||||
int8_t,
|
||||
int16_t,
|
||||
int32_t,
|
||||
int64_t,
|
||||
#if _CCCL_HAS_INT128()
|
||||
int128_t,
|
||||
#endif
|
||||
float,
|
||||
double,
|
||||
complex32,
|
||||
complex64>;
|
||||
|
||||
TEMPLATE_LIST_TEST_CASE("Generator seeds the data", "[gen]", types)
|
||||
{
|
||||
auto generator = generate(1 << 24, bit_entropy::_0_811);
|
||||
|
||||
const thrust::device_vector<TestType> vec_1 = generator;
|
||||
const thrust::device_vector<TestType> vec_2 = generator;
|
||||
|
||||
REQUIRE(vec_1.size() == vec_2.size());
|
||||
REQUIRE_FALSE(thrust::equal(vec_1.begin(), vec_1.end(), vec_2.begin()));
|
||||
}
|
||||
204
cccl_upstream/nvbench_helper/test/gen_uniform_distribution.cu
Normal file
204
cccl_upstream/nvbench_helper/test/gen_uniform_distribution.cu
Normal file
@@ -0,0 +1,204 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
|
||||
// SPDX-License-Identifier: BSD-3
|
||||
|
||||
#include <thrust/count.h>
|
||||
#include <thrust/device_vector.h>
|
||||
#include <thrust/host_vector.h>
|
||||
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
|
||||
#include <nvbench_helper.cuh>
|
||||
|
||||
#include <boost/math/distributions/chi_squared.hpp>
|
||||
#include <catch2/catch_template_test_macros.hpp>
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
#include <catch2/generators/catch_generators_all.hpp>
|
||||
|
||||
template <typename T>
|
||||
bool is_uniform(thrust::host_vector<T> data, T min, T max)
|
||||
{
|
||||
const double value_range = static_cast<double>(max) - min;
|
||||
const bool exact_binning = value_range < (1 << 20);
|
||||
const int number_of_bins = exact_binning ? static_cast<int>(max - min + 1) : static_cast<int>(std::sqrt(data.size()));
|
||||
thrust::host_vector<int> bins(number_of_bins, 0);
|
||||
|
||||
const double interval = value_range / static_cast<double>(number_of_bins);
|
||||
const double expected_count = static_cast<double>(data.size()) / number_of_bins;
|
||||
|
||||
for (T val : data)
|
||||
{
|
||||
int bin_index = exact_binning ? val - min : (val - static_cast<double>(min)) / interval;
|
||||
|
||||
if (bin_index >= 0 && bin_index < number_of_bins)
|
||||
{
|
||||
bins[bin_index]++;
|
||||
}
|
||||
}
|
||||
|
||||
double chi_square = 0.0;
|
||||
for (const auto& count : bins)
|
||||
{
|
||||
chi_square += std::pow(count - expected_count, 2) / expected_count;
|
||||
}
|
||||
|
||||
boost::math::chi_squared_distribution<double> chi_squared_dist(number_of_bins - 1);
|
||||
|
||||
const double confidence = 0.95;
|
||||
const double critical_value = boost::math::quantile(chi_squared_dist, confidence);
|
||||
|
||||
return chi_square <= critical_value;
|
||||
}
|
||||
|
||||
using types =
|
||||
nvbench::type_list<int8_t,
|
||||
int16_t,
|
||||
int32_t,
|
||||
int64_t,
|
||||
#if _CCCL_HAS_INT128()
|
||||
int128_t,
|
||||
#endif
|
||||
float,
|
||||
double>;
|
||||
|
||||
TEMPLATE_LIST_TEST_CASE("Generators produce uniformly distributed data", "[gen][uniform]", types)
|
||||
{
|
||||
const std::size_t elements = 1 << GENERATE_COPY(16, 20, 24, 28);
|
||||
const TestType min = ::cuda::std::numeric_limits<TestType>::min();
|
||||
const TestType max = ::cuda::std::numeric_limits<TestType>::max();
|
||||
|
||||
const thrust::device_vector<TestType> data = generate(elements, bit_entropy::_1_000, min, max);
|
||||
|
||||
REQUIRE(is_uniform<TestType>(data, min, max));
|
||||
}
|
||||
|
||||
struct complex_to_real_t
|
||||
{
|
||||
template <typename T>
|
||||
__host__ __device__ T operator()(const cuda::std::complex<T>& c) const
|
||||
{
|
||||
return c.real();
|
||||
}
|
||||
};
|
||||
|
||||
struct complex_to_imag_t
|
||||
{
|
||||
template <typename T>
|
||||
__host__ __device__ T operator()(const cuda::std::complex<T>& c) const
|
||||
{
|
||||
return c.imag();
|
||||
}
|
||||
};
|
||||
|
||||
using complex_value_types = nvbench::type_list<float, double>;
|
||||
|
||||
TEMPLATE_LIST_TEST_CASE("Generators produce uniformly distributed complex", "[gen]", complex_value_types)
|
||||
{
|
||||
using value_type = TestType;
|
||||
const auto min = ::cuda::std::numeric_limits<value_type>::min();
|
||||
const auto max = ::cuda::std::numeric_limits<value_type>::max();
|
||||
|
||||
const thrust::device_vector<cuda::std::complex<value_type>> data = generate(1 << 16);
|
||||
|
||||
thrust::device_vector<value_type> component(data.size());
|
||||
thrust::transform(data.begin(), data.end(), component.begin(), complex_to_real_t());
|
||||
REQUIRE(is_uniform<value_type>(component, min, max));
|
||||
|
||||
thrust::transform(data.begin(), data.end(), component.begin(), complex_to_imag_t());
|
||||
REQUIRE(is_uniform<value_type>(component, min, max));
|
||||
}
|
||||
|
||||
TEST_CASE("Generators produce uniformly distributed bools", "[gen]")
|
||||
{
|
||||
const thrust::device_vector<bool> data = generate(1 << 24, bit_entropy::_0_544);
|
||||
|
||||
const std::size_t falses = thrust::count(data.begin(), data.end(), false);
|
||||
const std::size_t trues = thrust::count(data.begin(), data.end(), true);
|
||||
|
||||
REQUIRE(falses > 0);
|
||||
REQUIRE(trues > 0);
|
||||
REQUIRE(falses + trues == data.size());
|
||||
|
||||
const double ratio = static_cast<double>(falses) / trues;
|
||||
REQUIRE(ratio > 0.7);
|
||||
}
|
||||
|
||||
using offsets = nvbench::type_list<uint32_t, uint64_t>;
|
||||
|
||||
TEMPLATE_LIST_TEST_CASE("Generators produce uniformly distributed offsets", "[gen]", offsets)
|
||||
{
|
||||
const std::size_t min_segment_size = 1;
|
||||
const std::size_t max_segment_size = 256;
|
||||
const std::size_t elements = 1 << GENERATE_COPY(16, 20, 24, 28);
|
||||
const thrust::device_vector<TestType> d_segments =
|
||||
generate.uniform.segment_offsets(elements, min_segment_size, max_segment_size);
|
||||
const thrust::host_vector<TestType> h_segments = d_segments;
|
||||
const std::size_t num_segments = h_segments.size() - 1;
|
||||
|
||||
std::size_t actual_elements = 0;
|
||||
thrust::host_vector<int> segment_sizes(num_segments);
|
||||
for (std::size_t sid = 0; sid < num_segments; sid++)
|
||||
{
|
||||
const TestType begin = h_segments[sid];
|
||||
const TestType end = h_segments[sid + 1];
|
||||
REQUIRE(begin <= end);
|
||||
|
||||
const TestType size = end - begin;
|
||||
REQUIRE(size >= min_segment_size);
|
||||
REQUIRE(size <= max_segment_size);
|
||||
|
||||
segment_sizes[sid] = size;
|
||||
actual_elements += size;
|
||||
}
|
||||
|
||||
REQUIRE(actual_elements == elements);
|
||||
REQUIRE(is_uniform<int>(std::move(segment_sizes), min_segment_size, max_segment_size));
|
||||
}
|
||||
|
||||
TEMPLATE_LIST_TEST_CASE("Generators produce uniformly distributed key segments", "[gen]", types)
|
||||
try
|
||||
{
|
||||
const std::size_t min_segment_size = 1;
|
||||
const std::size_t max_segment_size = 128;
|
||||
const std::size_t elements = 1 << GENERATE_COPY(16, 20, 24, 28);
|
||||
const thrust::device_vector<TestType> d_keys =
|
||||
generate.uniform.key_segments(elements, min_segment_size, max_segment_size);
|
||||
REQUIRE(d_keys.size() == elements);
|
||||
|
||||
const thrust::host_vector<TestType> h_keys = d_keys;
|
||||
|
||||
thrust::host_vector<std::size_t> segment_sizes;
|
||||
|
||||
TestType prev = h_keys[0];
|
||||
std::size_t length = 1;
|
||||
|
||||
for (std::size_t kid = 1; kid < elements; kid++)
|
||||
{
|
||||
TestType next = h_keys[kid];
|
||||
|
||||
if (next == prev)
|
||||
{
|
||||
length++;
|
||||
}
|
||||
else
|
||||
{
|
||||
REQUIRE(length >= min_segment_size);
|
||||
REQUIRE(length <= max_segment_size);
|
||||
|
||||
segment_sizes.push_back(length);
|
||||
|
||||
prev = next;
|
||||
length = 1;
|
||||
}
|
||||
}
|
||||
REQUIRE(length >= min_segment_size);
|
||||
REQUIRE(length <= max_segment_size);
|
||||
segment_sizes.push_back(length);
|
||||
|
||||
REQUIRE(is_uniform(std::move(segment_sizes), min_segment_size, max_segment_size));
|
||||
}
|
||||
catch (std::bad_alloc&)
|
||||
{
|
||||
// Skip test on OOM.
|
||||
}
|
||||
Reference in New Issue
Block a user