dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
900ae0b1efc19e46cd313eb3ba7eb694fa596f6b
project_6/cccl_upstream/cub/test/test_util.h
EngineX CI 56fd68e7dd [INFRA] Import NVIDIA/CCCL upstream as optimization reference library 
CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0
2026-07-30 09:35:51 +00:00

1545 lines
37 KiB
C++
Raw Blame History

// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved.
// SPDX-FileCopyrightText: Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved.
// SPDX-License-Identifier: BSD-3
#pragma once
#ifdef _WIN32
# include <windows.h>
# undef small // Windows is terrible for polluting macro namespace
#else
# include <sys/resource.h>
#endif
#include <cub/util_debug.cuh>
#include <cub/util_device.cuh>
#include <cub/util_macro.cuh>
#include <cub/util_math.cuh>
#include <cub/util_namespace.cuh>
#include <cub/util_ptx.cuh>
#include <cub/util_type.cuh>
#include <thrust/iterator/discard_iterator.h>
#include <cuda/std/algorithm>
#include <nv/target>
#include <cfloat>
#include <cmath>
#include <cstddef>
#include <cstdio>
#include <iostream>
#include <limits>
#include <sstream>
#include <string>
#include <vector>
#include "mersenne.h"
#include <c2h/catch2_test_helper.h>
#include <c2h/extended_types.h>
#include <c2h/test_util_vec.h>
/******************************************************************************
* Assertion macros
******************************************************************************/
/**
* Assert equals
*/
#define AssertEquals(a, b) \
if ((a) != (b)) \
{ \
std::cerr << "\n" << __FILE__ << ": " << __LINE__ << ": AssertEquals(" #a ", " #b ") failed.\n"; \
exit(1); \
}
#define AssertTrue(a) \
if (!(a)) \
{ \
std::cerr << "\n" << __FILE__ << ": " << __LINE__ << ": AssertTrue(" #a ") failed.\n"; \
exit(1); \
}
/******************************************************************************
* Command-line parsing functionality
******************************************************************************/
/**
* Utility for parsing command line arguments
*/
struct CommandLineArgs
{
std::vector<std::string> keys;
std::vector<std::string> values;
std::vector<std::string> args;
cudaDeviceProp deviceProp;
float device_giga_bandwidth;
std::size_t device_free_physmem;
std::size_t device_total_physmem;
/**
* Constructor
*/
CommandLineArgs(int argc, char** argv)
: keys(10)
, values(10)
{
using namespace std;
// Initialize mersenne generator
unsigned int mersenne_init[4] = {0x123, 0x234, 0x345, 0x456};
mersenne::init_by_array(mersenne_init, 4);
for (int i = 1; i < argc; i++)
{
string arg = argv[i];
if ((arg[0] != '-') || (arg[1] != '-'))
{
args.push_back(arg);
continue;
}
string key, val;
string::size_type pos = arg.find('=');
if (pos == string::npos)
{
key = string(arg, 2, arg.length() - 2);
val = "";
}
else
{
key = string(arg, 2, pos - 2);
val = string(arg, pos + 1, arg.length() - 1);
}
keys.push_back(key);
values.push_back(val);
}
}
/**
* Checks whether a flag "--<flag>" is present in the commandline
*/
bool CheckCmdLineFlag(const char* arg_name)
{
using namespace std;
for (const auto& key : keys)
{
if (key == string(arg_name))
{
return true;
}
}
return false;
}
/**
* Returns number of naked (non-flag and non-key-value) commandline parameters
*/
template <typename T>
int NumNakedArgs()
{
return static_cast<int>(args.size());
}
/**
* Returns the commandline parameter for a given index (not including flags)
*/
template <typename T>
void GetCmdLineArgument(std::size_t index, T& val)
{
using namespace std;
if (index < args.size())
{
istringstream str_stream(args[index]);
str_stream >> val;
}
}
/**
* Returns the value specified for a given commandline parameter --<flag>=<value>
*/
template <typename T>
void GetCmdLineArgument(const char* arg_name, T& val)
{
using namespace std;
for (std::size_t i = 0; i < keys.size(); ++i)
{
if (keys[i] == string(arg_name))
{
istringstream str_stream(values[i]);
str_stream >> val;
}
}
}
/**
* Returns the values specified for a given commandline parameter --<flag>=<value>,<value>*
*/
template <typename T>
void GetCmdLineArguments(const char* arg_name, std::vector<T>& vals)
{
using namespace std;
if (CheckCmdLineFlag(arg_name))
{
// Clear any default values
vals.clear();
// Recover from multi-value string
for (std::size_t i = 0; i < keys.size(); ++i)
{
if (keys[i] == string(arg_name))
{
string val_string(values[i]);
istringstream str_stream(val_string);
string::size_type old_pos = 0;
string::size_type new_pos = 0;
// Iterate comma-separated values
T val;
while ((new_pos = val_string.find(',', old_pos)) != string::npos)
{
if (new_pos != old_pos)
{
str_stream.width(static_cast<std::streamsize>(new_pos - old_pos));
str_stream >> val;
vals.push_back(val);
}
// skip over comma
str_stream.ignore(1);
old_pos = new_pos + 1;
}
// Read last value
str_stream >> val;
vals.push_back(val);
}
}
}
}
/**
* The number of pairs parsed
*/
int ParsedArgc()
{
return (int) keys.size();
}
/**
* Initialize device
*/
cudaError_t DeviceInit(int dev = -1)
{
cudaError_t error = cudaSuccess;
do
{
int deviceCount;
error = CubDebug(cudaGetDeviceCount(&deviceCount));
if (error)
{
break;
}
if (deviceCount == 0)
{
fprintf(stderr, "No devices supporting CUDA.\n");
exit(1);
}
if (dev < 0)
{
GetCmdLineArgument("device", dev);
}
if ((dev > deviceCount - 1) || (dev < 0))
{
dev = 0;
}
error = CubDebug(cudaSetDevice(dev));
if (error)
{
break;
}
CubDebugExit(cudaMemGetInfo(&device_free_physmem, &device_total_physmem));
int ptx_version = 0;
error = CubDebug(CUB_NS_QUALIFIER::PtxVersion(ptx_version));
if (error)
{
break;
}
error = CubDebug(cudaGetDeviceProperties(&deviceProp, dev));
if (error)
{
break;
}
if (deviceProp.major < 1)
{
fprintf(stderr, "Device does not support CUDA.\n");
exit(1);
}
int memoryClockRate{};
error = CubDebug(cudaDeviceGetAttribute(&memoryClockRate, cudaDevAttrMemoryClockRate, dev));
if (error)
{
break;
}
int memoryBusWidth{};
error = CubDebug(cudaDeviceGetAttribute(&memoryBusWidth, cudaDevAttrGlobalMemoryBusWidth, dev));
if (error)
{
break;
}
device_giga_bandwidth = float(memoryBusWidth) * static_cast<float>(memoryClockRate) * 2 / 8 / 1000 / 1000;
if (!CheckCmdLineFlag("quiet"))
{
printf(
"Using device %d: %s (PTX version %d, SM%d, %d SMs, "
"%lld free / %lld total MB physmem, "
"%.3f GB/s @ %d kHz mem clock, ECC %s)\n",
dev,
deviceProp.name,
ptx_version,
deviceProp.major * 100 + deviceProp.minor * 10,
deviceProp.multiProcessorCount,
(unsigned long long) device_free_physmem / 1024 / 1024,
(unsigned long long) device_total_physmem / 1024 / 1024,
device_giga_bandwidth,
memoryClockRate,
(deviceProp.ECCEnabled) ? "on" : "off");
fflush(stdout);
}
} while (false);
return error;
}
};
// Gets the amount of global memory of the current device.
inline std::size_t TotalGlobalMem()
{
int device = 0;
CubDebugExit(cudaGetDevice(&device));
std::size_t free_mem = 0, total_mem = 0;
CubDebugExit(cudaMemGetInfo(&free_mem, &total_mem));
return total_mem;
}
/******************************************************************************
* Random bits generator
******************************************************************************/
/**
* Generates random keys.
*
* We always take the second-order byte from rand() because the higher-order
* bits returned by rand() are commonly considered more uniformly distributed
* than the lower-order bits.
*
* We can decrease the entropy level of keys by adopting the technique
* of Thearling and Smith in which keys are computed from the bitwise AND of
* multiple random samples:
*
* entropy_reduction | Effectively-unique bits per key
* -----------------------------------------------------
* -1 | 0
* 0 | 32
* 1 | 25.95 (81%)
* 2 | 17.41 (54%)
* 3 | 10.78 (34%)
* 4 | 6.42 (20%)
* ... | ...
*
*/
template <typename K>
void RandomBits(K& key, int entropy_reduction = 0, int begin_bit = 0, int end_bit = sizeof(K) * 8)
{
constexpr int NUM_BYTES = sizeof(K);
constexpr int WORD_BYTES = sizeof(unsigned int);
constexpr int NUM_WORDS = (NUM_BYTES + WORD_BYTES - 1) / WORD_BYTES;
unsigned int word_buff[NUM_WORDS];
if (entropy_reduction == -1)
{
memset((void*) &key, 0, sizeof(key));
return;
}
if (end_bit < 0)
{
end_bit = sizeof(K) * 8;
}
while (true)
{
// Generate random word_buff
for (int j = 0; j < NUM_WORDS; j++)
{
int current_bit = j * WORD_BYTES * 8;
unsigned int word = 0xffffffff;
word &= 0xffffffff << ::cuda::std::max(0, begin_bit - current_bit);
word &= 0xffffffff >> ::cuda::std::max(0, (current_bit + (WORD_BYTES * 8)) - end_bit);
for (int i = 0; i <= entropy_reduction; i++)
{
// Grab some of the higher bits from rand (better entropy, supposedly)
word &= mersenne::genrand_int32();
}
word_buff[j] = word;
}
memcpy(&key, word_buff, sizeof(K));
K copy = key;
if (!c2h::isnan(copy))
{
break; // avoids NaNs when generating random floating point numbers
}
}
}
/// Randomly select number between [0:max)
template <typename T>
T RandomValue(T max)
{
unsigned int bits;
unsigned int max_int = (unsigned int) -1;
do
{
RandomBits(bits);
} while (bits == max_int);
return (T) ((double(bits) / double(max_int)) * double(max));
}
/******************************************************************************
* Test value initialization utilities
******************************************************************************/
/**
* Test problem generation options
*/
enum GenMode
{
UNIFORM, // Assign to '2', regardless of integer seed
INTEGER_SEED, // Assign to integer seed
RANDOM, // Assign to random, regardless of integer seed
RANDOM_BIT, // Assign to randomly chosen 0 or 1, regardless of integer seed
RANDOM_MINUS_PLUS_ZERO, // Assign to random, with some values being -0.0 or +0.0 patterns
};
/**
* Initialize value
*/
#pragma nv_exec_check_disable
template <typename T>
__host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, T& value, std::size_t index = 0)
{
// RandomBits is host-only.
NV_IF_ELSE_TARGET(
NV_IS_HOST,
({
switch (gen_mode)
{
case RANDOM:
RandomBits(value);
break;
case RANDOM_BIT: {
char c;
RandomBits(c, 0, 0, 1);
value = static_cast<T>((c > 0) ? 1 : -1);
break;
}
case RANDOM_MINUS_PLUS_ZERO: {
// Replace roughly 1/128 of values with -0.0 or +0.0, and
// generate the rest randomly
using UnsignedBits = typename CUB_NS_QUALIFIER::Traits<T>::UnsignedBits;
char c;
RandomBits(c);
if (c == 0)
{
// Replace 1/256 of values with +0.0 bit pattern
value = c2h::SafeBitCast<T>(UnsignedBits(0));
}
else if (c == 1)
{
// Replace 1/256 of values with -0.0 bit pattern
value = c2h::SafeBitCast<T>(UnsignedBits(UnsignedBits(1) << (sizeof(UnsignedBits) * 8) - 1));
}
else
{
// 127/128 of values are random
RandomBits(value);
}
break;
}
case UNIFORM:
value = 2;
break;
case INTEGER_SEED:
default:
value = static_cast<T>(index);
break;
}
}),
({
switch (gen_mode)
{
case RANDOM:
case RANDOM_BIT:
case RANDOM_MINUS_PLUS_ZERO:
_CubLog("%s\n", "cub::InitValue cannot generate random numbers on device.");
cuda::std::terminate();
break;
case UNIFORM:
value = 2;
break;
case INTEGER_SEED:
default:
value = static_cast<T>(index);
break;
}
}));
}
/**
* Initialize value (bool)
*/
#pragma nv_exec_check_disable
__host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, bool& value, std::size_t index = 0)
{
// RandomBits is host-only.
NV_IF_ELSE_TARGET(
NV_IS_HOST,
({
switch (gen_mode)
{
case RANDOM:
case RANDOM_BIT:
char c;
RandomBits(c, 0, 0, 1);
value = (c > 0);
break;
case UNIFORM:
value = true;
break;
case INTEGER_SEED:
default:
value = (index > 0);
break;
}
}),
({
switch (gen_mode)
{
case RANDOM:
case RANDOM_BIT:
case RANDOM_MINUS_PLUS_ZERO:
_CubLog("%s\n", "cub::InitValue cannot generate random numbers on device.");
cuda::std::terminate();
break;
case UNIFORM:
value = true;
break;
case INTEGER_SEED:
default:
value = (index > 0);
break;
}
}));
}
/**
* cub::NullType test initialization
*/
__host__ __device__ __forceinline__ void
InitValue(GenMode /* gen_mode */, CUB_NS_QUALIFIER::NullType& /* value */, std::size_t /* index */ = 0)
{}
/**
* cub::KeyValuePair<OffsetT, ValueT>test initialization
*/
#pragma nv_exec_check_disable
template <typename KeyT, typename ValueT>
__host__ __device__ __forceinline__ void
InitValue(GenMode gen_mode, CUB_NS_QUALIFIER::KeyValuePair<KeyT, ValueT>& value, std::size_t index = 0)
{
InitValue(gen_mode, value.value, index);
// This specialization only appears to be used by test_warp_scan.
// It initializes with uniform values and random keys, so we need to
// protect the call to the host-only RandomBits.
NV_IF_ELSE_TARGET(
NV_IS_HOST,
({
// Assign corresponding flag with a likelihood of the last bit
// being set with entropy-reduction level 3
RandomBits(value.key, 3);
value.key = (value.key & 0x1);
}),
({
_CubLog("%s\n", "cub::InitValue cannot generate random numbers on device.");
cuda::std::terminate();
}));
}
/******************************************************************************
* Comparison and ostream operators
******************************************************************************/
/**
* KeyValuePair ostream operator
*/
template <typename Key, typename Value>
std::ostream& operator<<(std::ostream& os, const CUB_NS_QUALIFIER::KeyValuePair<Key, Value>& val)
{
os << '(' << CoutCast(val.key) << ',' << CoutCast(val.value) << ')';
return os;
}
#if _CCCL_HAS_NVFP16()
inline std::ostream& operator<<(std::ostream& stream, const __half2& value)
{
stream << "(" << value.x << "," << value.y << ")";
return stream;
}
#endif // _CCCL_HAS_HALF
#if _CCCL_HAS_NVBF16()
inline std::ostream& operator<<(std::ostream& stream, const __nv_bfloat162& value)
{
stream << "(" << value.x << "," << value.y << ")";
return stream;
}
#endif // _CCCL_HAS_NVBF16
#if TEST_INT128()
inline std::ostream& operator<<(std::ostream& os, __uint128_t val)
{
constexpr int max_digits = 40;
char buffer[max_digits] = {};
char* digit = buffer + max_digits;
static constexpr char ascii[] = "0123456789";
do
{
digit--;
*digit = ascii[val % 10];
val /= 10;
} while (val != 0);
for (; digit != buffer + max_digits; digit++)
{
os << *digit;
}
return os;
}
inline std::ostream& operator<<(std::ostream& os, __int128_t val)
{
if (val < 0)
{
__uint128_t tmp = -val;
os << '-' << tmp;
}
else
{
__uint128_t tmp = val;
os << tmp;
}
return os;
}
#endif
/******************************************************************************
* Comparison and ostream operators for CUDA vector types
******************************************************************************/
/**
* Vector1 overloads
*/
#define CUB_VEC_OVERLOAD_1_OLD(T) \
/* Test initialization */ \
__host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, T& value, std::size_t index = 0) \
{ \
InitValue(gen_mode, value.x, index); \
}
/**
* Vector2 overloads
*/
#define CUB_VEC_OVERLOAD_2_OLD(T) \
/* Test initialization */ \
__host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, T& value, std::size_t index = 0) \
{ \
InitValue(gen_mode, value.x, index); \
InitValue(gen_mode, value.y, index); \
}
/**
* Vector3 overloads
*/
#define CUB_VEC_OVERLOAD_3_OLD(T) \
/* Test initialization */ \
__host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, T& value, std::size_t index = 0) \
{ \
InitValue(gen_mode, value.x, index); \
InitValue(gen_mode, value.y, index); \
InitValue(gen_mode, value.z, index); \
}
/**
* Vector4 overloads
*/
#define CUB_VEC_OVERLOAD_4_OLD(T) \
/* Test initialization */ \
__host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, T& value, std::size_t index = 0) \
{ \
InitValue(gen_mode, value.x, index); \
InitValue(gen_mode, value.y, index); \
InitValue(gen_mode, value.z, index); \
InitValue(gen_mode, value.w, index); \
}
/**
* All vector overloads
*/
#define CUB_VEC_OVERLOAD_OLD(COMPONENT_T) \
CUB_VEC_OVERLOAD_1_OLD(COMPONENT_T##1) \
CUB_VEC_OVERLOAD_2_OLD(COMPONENT_T##2) \
CUB_VEC_OVERLOAD_3_OLD(COMPONENT_T##3) \
CUB_VEC_OVERLOAD_4_OLD(COMPONENT_T##4)
/**
* Define for types
*/
CUB_VEC_OVERLOAD_OLD(char)
CUB_VEC_OVERLOAD_OLD(short)
CUB_VEC_OVERLOAD_OLD(int)
_CCCL_SUPPRESS_DEPRECATED_PUSH
_CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
CUB_VEC_OVERLOAD_OLD(long)
CUB_VEC_OVERLOAD_OLD(longlong)
_CCCL_SUPPRESS_DEPRECATED_POP
#if _CCCL_CTK_AT_LEAST(13, 0)
CUB_VEC_OVERLOAD_4_OLD(long4_16a)
CUB_VEC_OVERLOAD_4_OLD(long4_32a)
CUB_VEC_OVERLOAD_4_OLD(longlong4_16a)
CUB_VEC_OVERLOAD_4_OLD(longlong4_32a)
#endif // _CCCL_CTK_AT_LEAST(13, 0)
CUB_VEC_OVERLOAD_OLD(uchar)
CUB_VEC_OVERLOAD_OLD(ushort)
CUB_VEC_OVERLOAD_OLD(uint)
_CCCL_SUPPRESS_DEPRECATED_PUSH
_CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
CUB_VEC_OVERLOAD_OLD(ulong)
CUB_VEC_OVERLOAD_OLD(ulonglong)
_CCCL_SUPPRESS_DEPRECATED_POP
#if _CCCL_CTK_AT_LEAST(13, 0)
CUB_VEC_OVERLOAD_4_OLD(ulong4_16a)
CUB_VEC_OVERLOAD_4_OLD(ulong4_32a)
CUB_VEC_OVERLOAD_4_OLD(ulonglong4_16a)
CUB_VEC_OVERLOAD_4_OLD(ulonglong4_32a)
#endif // _CCCL_CTK_AT_LEAST(13, 0)
CUB_VEC_OVERLOAD_OLD(float)
_CCCL_SUPPRESS_DEPRECATED_PUSH
_CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
CUB_VEC_OVERLOAD_OLD(double)
_CCCL_SUPPRESS_DEPRECATED_POP
#if _CCCL_CTK_AT_LEAST(13, 0)
CUB_VEC_OVERLOAD_4_OLD(double4_16a)
CUB_VEC_OVERLOAD_4_OLD(double4_32a)
#endif // _CCCL_CTK_AT_LEAST(13, 0)
//---------------------------------------------------------------------
// Complex data type TestFoo
//---------------------------------------------------------------------
/**
* TestFoo complex data type
*/
struct TestFoo
{
using x_t = long long;
using y_t = int;
using z_t = short;
using w_t = char;
x_t x;
y_t y;
z_t z;
w_t w;
// Factory
static __host__ __device__ __forceinline__ TestFoo MakeTestFoo(long long x, int y, short z, char w)
{
TestFoo retval = {x, y, z, w};
return retval;
}
// Assignment from int operator
__host__ __device__ __forceinline__ TestFoo& operator=(int b)
{
x = static_cast<x_t>(b);
y = static_cast<y_t>(b);
z = static_cast<z_t>(b);
w = static_cast<w_t>(b);
return *this;
}
// Summation operator
__host__ __device__ __forceinline__ TestFoo operator+(const TestFoo& b) const
{
return MakeTestFoo(x + b.x, y + b.y, static_cast<short>(z + b.z), static_cast<char>(w + b.w));
}
// Inequality operator
__host__ __device__ __forceinline__ bool operator!=(const TestFoo& b) const
{
return (x != b.x) || (y != b.y) || (z != b.z) || (w != b.w);
}
// Equality operator
__host__ __device__ __forceinline__ bool operator==(const TestFoo& b) const
{
return (x == b.x) && (y == b.y) && (z == b.z) && (w == b.w);
}
// Less than operator
__host__ __device__ __forceinline__ bool operator<(const TestFoo& b) const
{
if (x < b.x)
{
return true;
}
else if (b.x < x)
{
return false;
}
if (y < b.y)
{
return true;
}
else if (b.y < y)
{
return false;
}
if (z < b.z)
{
return true;
}
else if (b.z < z)
{
return false;
}
return w < b.w;
}
// Greater than operator
__host__ __device__ __forceinline__ bool operator>(const TestFoo& b) const
{
if (x > b.x)
{
return true;
}
else if (b.x > x)
{
return false;
}
if (y > b.y)
{
return true;
}
else if (b.y > y)
{
return false;
}
if (z > b.z)
{
return true;
}
else if (b.z > z)
{
return false;
}
return w > b.w;
}
};
/**
* TestFoo ostream operator
*/
inline std::ostream& operator<<(std::ostream& os, const TestFoo& val)
{
os << '(' << val.x << ',' << val.y << ',' << val.z << ',' << CoutCast(val.w) << ')';
return os;
}
/**
* TestFoo test initialization
*/
__host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, TestFoo& value, std::size_t index = 0)
{
InitValue(gen_mode, value.x, index);
InitValue(gen_mode, value.y, index);
InitValue(gen_mode, value.z, index);
InitValue(gen_mode, value.w, index);
}
template <>
class ::cuda::std::numeric_limits<TestFoo>
{
public:
static constexpr bool is_specialized = true;
__host__ __device__ static TestFoo max()
{
return TestFoo::MakeTestFoo(
numeric_limits<long long>::max(),
numeric_limits<int>::max(),
numeric_limits<short>::max(),
numeric_limits<char>::max());
}
__host__ __device__ static TestFoo lowest()
{
return TestFoo::MakeTestFoo(
numeric_limits<long long>::lowest(),
numeric_limits<int>::lowest(),
numeric_limits<short>::lowest(),
numeric_limits<char>::lowest());
}
};
//---------------------------------------------------------------------
// Complex data type TestBar (with optimizations for fence-free warp-synchrony)
//---------------------------------------------------------------------
/**
* TestBar complex data type
*/
struct TestBar
{
long long x;
int y;
// Constructor
__host__ __device__ __forceinline__ TestBar()
: x(0)
, y(0)
{}
// Constructor
__host__ __device__ __forceinline__ TestBar(int b)
: x(b)
, y(b)
{}
// Constructor
__host__ __device__ __forceinline__ TestBar(long long x, int y)
: x(x)
, y(y)
{}
// Assignment from int operator
__host__ __device__ __forceinline__ TestBar& operator=(int b)
{
x = b;
y = b;
return *this;
}
// Summation operator
__host__ __device__ __forceinline__ TestBar operator+(const TestBar& b) const
{
return TestBar(x + b.x, y + b.y);
}
// Inequality operator
__host__ __device__ __forceinline__ bool operator!=(const TestBar& b) const
{
return (x != b.x) || (y != b.y);
}
// Equality operator
__host__ __device__ __forceinline__ bool operator==(const TestBar& b) const
{
return (x == b.x) && (y == b.y);
}
// Less than operator
__host__ __device__ __forceinline__ bool operator<(const TestBar& b) const
{
if (x < b.x)
{
return true;
}
else if (b.x < x)
{
return false;
}
return y < b.y;
}
// Greater than operator
__host__ __device__ __forceinline__ bool operator>(const TestBar& b) const
{
if (x > b.x)
{
return true;
}
else if (b.x > x)
{
return false;
}
return y > b.y;
}
};
/**
* TestBar ostream operator
*/
inline std::ostream& operator<<(std::ostream& os, const TestBar& val)
{
os << '(' << val.x << ',' << val.y << ')';
return os;
}
/**
* TestBar test initialization
*/
__host__ __device__ __forceinline__ void InitValue(GenMode gen_mode, TestBar& value, std::size_t index = 0)
{
InitValue(gen_mode, value.x, index);
InitValue(gen_mode, value.y, index);
}
template <>
class cuda::std::numeric_limits<TestBar>
{
public:
static constexpr bool is_specialized = true;
__host__ __device__ static TestBar max()
{
return TestBar(numeric_limits<long long>::max(), numeric_limits<int>::max());
}
__host__ __device__ static TestBar lowest()
{
return TestBar(numeric_limits<long long>::lowest(), numeric_limits<int>::lowest());
}
};
/******************************************************************************
* Helper routines for list comparison and display
******************************************************************************/
/**
* Compares the equivalence of two arrays
*/
template <typename S, typename T, typename OffsetT>
int CompareResults(T* computed, S* reference, OffsetT len, bool verbose = true)
{
for (OffsetT i = 0; i < len; i++)
{
if (computed[i] != reference[i])
{
if (verbose)
{
std::cout << "INCORRECT: [" << i << "]: " << CoutCast(computed[i]) << " != " << CoutCast(reference[i]);
}
return 1;
}
}
return 0;
}
/**
* Compares the equivalence of two arrays
*/
template <typename OffsetT>
int CompareResults(float* computed, float* reference, OffsetT len, bool verbose = true)
{
for (OffsetT i = 0; i < len; i++)
{
if (computed[i] != reference[i])
{
float difference = std::abs(computed[i] - reference[i]);
float fraction = difference / std::abs(reference[i]);
if (fraction > 0.00015)
{
if (verbose)
{
std::cout
<< "INCORRECT: [" << i << "]: "
<< "(computed) " << CoutCast(computed[i]) << " != " << CoutCast(reference[i])
<< " (difference:" << difference << ", fraction: " << fraction << ")";
}
return 1;
}
}
}
return 0;
}
/**
* Compares the equivalence of two arrays
*/
template <typename OffsetT>
int CompareResults(
CUB_NS_QUALIFIER::NullType* computed, CUB_NS_QUALIFIER::NullType* reference, OffsetT len, bool verbose = true)
{
return 0;
}
/**
* Compares the equivalence of two arrays
*/
template <typename OffsetT>
int CompareResults(double* computed, double* reference, OffsetT len, bool verbose = true)
{
for (OffsetT i = 0; i < len; i++)
{
if (computed[i] != reference[i])
{
double difference = std::abs(computed[i] - reference[i]);
double fraction = difference / std::abs(reference[i]);
if (fraction > 0.00015)
{
if (verbose)
{
std::cout << "INCORRECT: [" << i << "]: " << CoutCast(computed[i]) << " != " << CoutCast(reference[i])
<< " (difference:" << difference << ", fraction: " << fraction << ")";
}
return 1;
}
}
}
return 0;
}
/**
* Verify the contents of a device array match those
* of a host array
*/
inline int CompareDeviceResults(
CUB_NS_QUALIFIER::NullType* /* h_reference */,
CUB_NS_QUALIFIER::NullType* /* d_data */,
std::size_t /* num_items */,
bool /* verbose */ = true,
bool /* display_data */ = false)
{
return 0;
}
/**
* Verify the contents of a device array match those
* of a host array
*/
template <typename S, typename OffsetT>
int CompareDeviceResults(
S* /*h_reference*/,
THRUST_NS_QUALIFIER::discard_iterator<OffsetT> /*d_data*/,
std::size_t /*num_items*/,
bool /*verbose*/ = true,
bool /*display_data*/ = false)
{
return 0;
}
/**
* Verify the contents of a device array match those
* of a host array
*/
template <typename S, typename T>
int CompareDeviceResults(
S* h_reference, T* d_data, std::size_t num_items, bool verbose = true, bool display_data = false)
{
if (num_items == 0)
{
return 0;
}
// Allocate array on host
T* h_data = (T*) malloc(num_items * sizeof(T));
// Copy data back
cudaMemcpy(h_data, d_data, sizeof(T) * num_items, cudaMemcpyDeviceToHost);
// Display data
if (display_data)
{
printf("Reference:\n");
for (std::size_t i = 0; i < num_items; i++)
{
std::cout << CoutCast(h_reference[i]) << ", ";
}
printf("\n\nComputed:\n");
for (std::size_t i = 0; i < num_items; i++)
{
std::cout << CoutCast(h_data[i]) << ", ";
}
printf("\n\n");
}
// Check
int retval = CompareResults(h_data, h_reference, num_items, verbose);
// Cleanup
if (h_data)
{
free(h_data);
}
return retval;
}
/**
* Verify the contents of a device array match those
* of a device array
*/
template <typename T>
int CompareDeviceDeviceResults(
T* d_reference, T* d_data, std::size_t num_items, bool verbose = true, bool display_data = false)
{
// Allocate array on host
T* h_reference = (T*) malloc(num_items * sizeof(T));
T* h_data = (T*) malloc(num_items * sizeof(T));
// Copy data back
cudaMemcpy(h_reference, d_reference, sizeof(T) * num_items, cudaMemcpyDeviceToHost);
cudaMemcpy(h_data, d_data, sizeof(T) * num_items, cudaMemcpyDeviceToHost);
// Display data
if (display_data)
{
printf("Reference:\n");
for (std::size_t i = 0; i < num_items; i++)
{
std::cout << CoutCast(h_reference[i]) << ", ";
}
printf("\n\nComputed:\n");
for (std::size_t i = 0; i < num_items; i++)
{
std::cout << CoutCast(h_data[i]) << ", ";
}
printf("\n\n");
}
// Check
int retval = CompareResults(h_data, h_reference, num_items, verbose);
// Cleanup
if (h_reference)
{
free(h_reference);
}
if (h_data)
{
free(h_data);
}
return retval;
}
/**
* Print the contents of a host array
*/
inline void DisplayResults(CUB_NS_QUALIFIER::NullType* /* h_data */, std::size_t /* num_items */) {}
/**
* Print the contents of a host array
*/
template <typename InputIteratorT>
void DisplayResults(InputIteratorT h_data, std::size_t num_items)
{
// Display data
for (std::size_t i = 0; i < num_items; i++)
{
std::cout << CoutCast(h_data[i]) << ", ";
}
printf("\n");
}
/**
* Print the contents of a device array
*/
template <typename T>
void DisplayDeviceResults(T* d_data, std::size_t num_items)
{
// Allocate array on host
T* h_data = (T*) malloc(num_items * sizeof(T));
// Copy data back
cudaMemcpy(h_data, d_data, sizeof(T) * num_items, cudaMemcpyDeviceToHost);
DisplayResults(h_data, num_items);
// Cleanup
if (h_data)
{
free(h_data);
}
}
/******************************************************************************
* Segment descriptor generation
******************************************************************************/
/**
* Initialize segments
*/
template <typename OffsetT>
void InitializeSegments(OffsetT num_items, int num_segments, OffsetT* h_segment_offsets, bool verbose = false)
{
if (num_segments <= 0)
{
return;
}
OffsetT expected_segment_length = ::cuda::ceil_div(num_items, OffsetT(num_segments));
OffsetT offset = 0;
for (int i = 0; i < num_segments; ++i)
{
h_segment_offsets[i] = offset;
OffsetT segment_length = RandomValue((expected_segment_length * 2) + 1);
offset += segment_length;
offset = ::cuda::std::min(offset, num_items);
}
h_segment_offsets[num_segments] = num_items;
if (verbose)
{
printf("Segment offsets: ");
DisplayResults(h_segment_offsets, num_segments + 1);
}
}
/******************************************************************************
* Timing
******************************************************************************/
struct CpuTimer
{
#ifdef _WIN32
LARGE_INTEGER ll_freq;
LARGE_INTEGER ll_start;
LARGE_INTEGER ll_stop;
CpuTimer()
{
QueryPerformanceFrequency(&ll_freq);
}
void Start()
{
QueryPerformanceCounter(&ll_start);
}
void Stop()
{
QueryPerformanceCounter(&ll_stop);
}
float ElapsedMillis()
{
double start = double(ll_start.QuadPart) / double(ll_freq.QuadPart);
double stop = double(ll_stop.QuadPart) / double(ll_freq.QuadPart);
return float((stop - start) * 1000);
}
#else
rusage start;
rusage stop;
void Start()
{
getrusage(RUSAGE_SELF, &start);
}
void Stop()
{
getrusage(RUSAGE_SELF, &stop);
}
float ElapsedMillis()
{
float sec = static_cast<float>(stop.ru_utime.tv_sec - start.ru_utime.tv_sec);
float usec = static_cast<float>(stop.ru_utime.tv_usec - start.ru_utime.tv_usec);
return (sec * 1000) + (usec / 1000);
}
#endif
};
struct GpuTimer
{
cudaEvent_t start;
cudaEvent_t stop;
GpuTimer()
{
cudaEventCreate(&start);
cudaEventCreate(&stop);
}
~GpuTimer()
{
cudaEventDestroy(start);
cudaEventDestroy(stop);
}
void Start()
{
cudaEventRecord(start, nullptr);
}
void Stop()
{
cudaEventRecord(stop, nullptr);
}
float ElapsedMillis()
{
float elapsed;
cudaEventSynchronize(stop);
cudaEventElapsedTime(&elapsed, start, stop);
return elapsed;
}
};
template <int ELEMENTS_PER_OBJECT_ = 128>
struct HugeDataType
{
static constexpr int ELEMENTS_PER_OBJECT = ELEMENTS_PER_OBJECT_;
__device__ __host__ HugeDataType()
{
for (int i = 0; i < ELEMENTS_PER_OBJECT; i++)
{
data[i] = 0;
}
}
__device__ __host__ HugeDataType(const HugeDataType& rhs)
{
for (int i = 0; i < ELEMENTS_PER_OBJECT; i++)
{
data[i] = rhs.data[i];
}
}
explicit __device__ __host__ HugeDataType(int val)
{
for (int i = 0; i < ELEMENTS_PER_OBJECT; i++)
{
data[i] = val;
}
}
__device__ __host__ HugeDataType& operator=(const HugeDataType& rhs)
{
if (this != &rhs)
{
for (int i = 0; i < ELEMENTS_PER_OBJECT; i++)
{
data[i] = rhs.data[i];
}
}
return *this;
}
int data[ELEMENTS_PER_OBJECT];
};
template <int ELEMENTS_PER_OBJECT>
inline __device__ __host__ bool
operator==(const HugeDataType<ELEMENTS_PER_OBJECT>& lhs, const HugeDataType<ELEMENTS_PER_OBJECT>& rhs)
{
for (int i = 0; i < ELEMENTS_PER_OBJECT; i++)
{
if (lhs.data[i] != rhs.data[i])
{
return false;
}
}
return true;
}
template <int ELEMENTS_PER_OBJECT>
inline __device__ __host__ bool
operator<(const HugeDataType<ELEMENTS_PER_OBJECT>& lhs, const HugeDataType<ELEMENTS_PER_OBJECT>& rhs)
{
for (int i = 0; i < ELEMENTS_PER_OBJECT; i++)
{
if (lhs.data[i] < rhs.data[i])
{
return true;
}
}
return false;
}
template <typename DataType, int ELEMENTS_PER_OBJECT>
__device__ __host__ bool operator!=(const HugeDataType<ELEMENTS_PER_OBJECT>& lhs, const DataType& rhs)
{
for (int i = 0; i < ELEMENTS_PER_OBJECT; i++)
{
if (lhs.data[i] != rhs)
{
return true;
}
}
return false;
}
template <int ELEMENTS_PER_OBJECT>
std::ostream& operator<<(std::ostream& os, const HugeDataType<ELEMENTS_PER_OBJECT>& val)
{
os << '(';
for (int i = 0; i < ELEMENTS_PER_OBJECT; i++)
{
os << CoutCast(val.data[i]);
if (i < ELEMENTS_PER_OBJECT - 1)
{
os << ',';
}
}
os << ')';
return os;
}
Reference in New Issue View Git Blame Copy Permalink