[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___THREAD_THREADING_SUPPORT_H
|
||||
#define _CUDA_STD___THREAD_THREADING_SUPPORT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__chrono/duration.h>
|
||||
#include <cuda/std/__chrono/high_resolution_clock.h>
|
||||
|
||||
#if defined(_CCCL_HAS_THREAD_API_EXTERNAL)
|
||||
# include <cuda/std/__thread/threading_support_external.h>
|
||||
#endif // _CCCL_HAS_THREAD_API_EXTERNAL
|
||||
|
||||
#if defined(_CCCL_HAS_THREAD_API_CUDA)
|
||||
# include <cuda/std/__thread/threading_support_cuda.h>
|
||||
#elif defined(_CCCL_HAS_THREAD_API_PTHREAD)
|
||||
# include <cuda/std/__thread/threading_support_pthread.h>
|
||||
#elif defined(_CCCL_HAS_THREAD_API_WIN32)
|
||||
# include <cuda/std/__thread/threading_support_win32.h>
|
||||
#else // ^^^ _CCCL_HAS_THREAD_API_WIN32 ^^^ / vvv Unknown Thread API vvv
|
||||
# error "Unknown Thread API"
|
||||
#endif // Unknown Thread API
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
#define _LIBCUDACXX_POLLING_COUNT 16
|
||||
|
||||
#if _CCCL_HOST_ARCH(ARM64) && _CCCL_OS(LINUX)
|
||||
# define __LIBCUDACXX_ASM_THREAD_YIELD (asm volatile("yield" :: :);)
|
||||
#elif _CCCL_HOST_ARCH(X86_64) && _CCCL_OS(LINUX)
|
||||
# define __LIBCUDACXX_ASM_THREAD_YIELD (asm volatile("pause" :: :);)
|
||||
#else // ^^^ _CCCL_HOST_ARCH(X86_64) ^^^ / vvv ! _CCCL_HOST_ARCH(X86_64) vvv
|
||||
# define __LIBCUDACXX_ASM_THREAD_YIELD (;)
|
||||
#endif // ! _CCCL_HOST_ARCH(X86_64)
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline void __cccl_thread_yield_processor()
|
||||
{
|
||||
NV_IF_TARGET(NV_IS_HOST, __LIBCUDACXX_ASM_THREAD_YIELD)
|
||||
}
|
||||
|
||||
template <class _Fn>
|
||||
_CCCL_HOST_DEVICE_API inline bool __cccl_thread_poll_with_backoff(
|
||||
_Fn&& __f, ::cuda::std::chrono::nanoseconds __max = ::cuda::std::chrono::nanoseconds::zero())
|
||||
{
|
||||
::cuda::std::chrono::high_resolution_clock::time_point const __start =
|
||||
::cuda::std::chrono::high_resolution_clock::now();
|
||||
for (int __count = 0;;)
|
||||
{
|
||||
if (__f())
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if (__count < _LIBCUDACXX_POLLING_COUNT)
|
||||
{
|
||||
if (__count > (_LIBCUDACXX_POLLING_COUNT >> 1))
|
||||
{
|
||||
::cuda::std::__cccl_thread_yield_processor();
|
||||
}
|
||||
__count += 1;
|
||||
continue;
|
||||
}
|
||||
::cuda::std::chrono::high_resolution_clock::duration const __elapsed =
|
||||
::cuda::std::chrono::high_resolution_clock::now() - __start;
|
||||
if (__max != ::cuda::std::chrono::nanoseconds::zero() && __max < __elapsed)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
::cuda::std::chrono::nanoseconds const __step = __elapsed / 4;
|
||||
if (__step >= ::cuda::std::chrono::milliseconds(1))
|
||||
{
|
||||
::cuda::std::__cccl_thread_sleep_for(::cuda::std::chrono::milliseconds(1));
|
||||
}
|
||||
else if (__step >= ::cuda::std::chrono::microseconds(10))
|
||||
{
|
||||
::cuda::std::__cccl_thread_sleep_for(__step);
|
||||
}
|
||||
else
|
||||
{
|
||||
::cuda::std::__cccl_thread_yield();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_STD___THREAD_THREADING_SUPPORT_H
|
||||
@@ -0,0 +1,47 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___THREAD_THREADING_SUPPORT_CUDA_H
|
||||
#define _CUDA_STD___THREAD_THREADING_SUPPORT_CUDA_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if defined(_CCCL_HAS_THREAD_API_CUDA)
|
||||
|
||||
# include <cuda/std/__chrono/duration.h>
|
||||
# include <cuda/std/climits>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline void __cccl_thread_yield() {}
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline void __cccl_thread_sleep_for(::cuda::std::chrono::nanoseconds __ns){
|
||||
NV_IF_TARGET(NV_PROVIDES_SM_70, ({
|
||||
auto const __step = __ns.count();
|
||||
_CCCL_ASSERT(__step < numeric_limits<unsigned>::max(), "invalid nanoseconds count");
|
||||
::__nanosleep((unsigned) __step);
|
||||
}))}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_THREAD_API_CUDA
|
||||
|
||||
#endif // _CUDA_STD___THREAD_THREADING_SUPPORT_CUDA_H
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___THREAD_THREADING_SUPPORT_EXTERNAL_H
|
||||
#define _CUDA_STD___THREAD_THREADING_SUPPORT_EXTERNAL_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if defined(_CCCL_HAS_THREAD_API_EXTERNAL)
|
||||
|
||||
# include <cuda/std/__chrono/duration.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline void __cccl_thread_yield();
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline void __cccl_thread_sleep_for(::cuda::std::chrono::nanoseconds __ns);
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_THREAD_API_EXTERNAL
|
||||
|
||||
#endif // _CUDA_STD___THREAD_THREADING_SUPPORT_EXTERNAL_H
|
||||
@@ -0,0 +1,143 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___THREAD_THREADING_SUPPORT_PTHREAD_H
|
||||
#define _CUDA_STD___THREAD_THREADING_SUPPORT_PTHREAD_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if defined(_CCCL_HAS_THREAD_API_PTHREAD)
|
||||
|
||||
# include <cuda/std/__chrono/duration.h>
|
||||
# include <cuda/std/__utility/cmp.h>
|
||||
# include <cuda/std/climits>
|
||||
# include <cuda/std/ctime>
|
||||
|
||||
# include <errno.h>
|
||||
# include <pthread.h>
|
||||
# include <sched.h>
|
||||
# include <semaphore.h>
|
||||
# if defined(__linux__)
|
||||
# include <unistd.h>
|
||||
|
||||
# include <linux/futex.h>
|
||||
# include <sys/syscall.h>
|
||||
# endif // __linux__
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
// Mutex
|
||||
using __cccl_mutex_t = pthread_mutex_t;
|
||||
# define _LIBCUDACXX_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
|
||||
|
||||
using __cccl_recursive_mutex_t = pthread_mutex_t;
|
||||
|
||||
// Condition Variable
|
||||
using __cccl_condvar_t = pthread_cond_t;
|
||||
# define _LIBCUDACXX_CONDVAR_INITIALIZER PTHREAD_COND_INITIALIZER
|
||||
|
||||
// Semaphore
|
||||
using __cccl_semaphore_t = sem_t;
|
||||
# define _LIBCUDACXX_SEMAPHORE_MAX SEM_VALUE_MAX
|
||||
|
||||
// Execute once
|
||||
using __cccl_exec_once_flag = pthread_once_t;
|
||||
# define _LIBCUDACXX_EXEC_ONCE_INITIALIZER PTHREAD_ONCE_INIT
|
||||
|
||||
// Thread id
|
||||
using __cccl_thread_id = pthread_t;
|
||||
|
||||
// Thread
|
||||
# define _LIBCUDACXX_NULL_THREAD 0U
|
||||
|
||||
using __cccl_thread_t = pthread_t;
|
||||
|
||||
// Thread Local Storage
|
||||
using __cccl_tls_key = pthread_key_t;
|
||||
|
||||
# define _LIBCUDACXX_TLS_DESTRUCTOR_CC
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr timespec __cccl_to_timespec(const ::cuda::std::chrono::nanoseconds& __ns)
|
||||
{
|
||||
constexpr auto __ts_sec_max = numeric_limits<time_t>::max();
|
||||
|
||||
timespec __ts{};
|
||||
const auto __s = ::cuda::std::chrono::duration_cast<chrono::seconds>(__ns);
|
||||
|
||||
if (::cuda::std::cmp_less(__s.count(), __ts_sec_max))
|
||||
{
|
||||
__ts.tv_sec = static_cast<time_t>(__s.count());
|
||||
__ts.tv_nsec = static_cast<decltype(__ts.tv_nsec)>((__ns - __s).count());
|
||||
}
|
||||
else
|
||||
{
|
||||
__ts.tv_sec = __ts_sec_max;
|
||||
__ts.tv_nsec = 999'999'999;
|
||||
}
|
||||
return __ts;
|
||||
}
|
||||
|
||||
// Semaphore
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline bool __cccl_semaphore_init(__cccl_semaphore_t* __sem, int __init)
|
||||
{
|
||||
return sem_init(__sem, 0, __init) == 0;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline bool __cccl_semaphore_destroy(__cccl_semaphore_t* __sem)
|
||||
{
|
||||
return sem_destroy(__sem) == 0;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline bool __cccl_semaphore_post(__cccl_semaphore_t* __sem)
|
||||
{
|
||||
return sem_post(__sem) == 0;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline bool __cccl_semaphore_wait(__cccl_semaphore_t* __sem)
|
||||
{
|
||||
return sem_wait(__sem) == 0;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline bool
|
||||
__cccl_semaphore_wait_timed(__cccl_semaphore_t* __sem, ::cuda::std::chrono::nanoseconds const& __ns)
|
||||
{
|
||||
const auto __ts = __cccl_to_timespec(__ns);
|
||||
return sem_timedwait(__sem, &__ts) == 0;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline void __cccl_thread_yield()
|
||||
{
|
||||
sched_yield();
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline void __cccl_thread_sleep_for(::cuda::std::chrono::nanoseconds __ns)
|
||||
{
|
||||
auto __ts = __cccl_to_timespec(__ns);
|
||||
while (nanosleep(&__ts, &__ts) == -1 && errno == EINTR)
|
||||
;
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // !_CCCL_HAS_THREAD_API_PTHREAD
|
||||
|
||||
#endif // _CUDA_STD___THREAD_THREADING_SUPPORT_PTHREAD_H
|
||||
@@ -0,0 +1,87 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___THREAD_THREADING_SUPPORT_WIN32_H
|
||||
#define _CUDA_STD___THREAD_THREADING_SUPPORT_WIN32_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if defined(_CCCL_HAS_THREAD_API_WIN32)
|
||||
|
||||
# include <cuda/std/__chrono/duration.h>
|
||||
|
||||
# include <process.h>
|
||||
# include <windows.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
// Mutex
|
||||
using __cccl_mutex_t = void*;
|
||||
# define _LIBCUDACXX_MUTEX_INITIALIZER 0
|
||||
|
||||
# if _CCCL_HOST_ARCH(ARM64) || _CCCL_HOST_ARCH(X86_64)
|
||||
using __cccl_recursive_mutex_t = void* [5];
|
||||
# else
|
||||
# error Unsupported architecture
|
||||
# endif
|
||||
|
||||
// Condition Variable
|
||||
using __cccl_condvar_t = void*;
|
||||
# define _LIBCUDACXX_CONDVAR_INITIALIZER 0
|
||||
|
||||
// Semaphore
|
||||
using __cccl_semaphore_t = void*;
|
||||
|
||||
// Execute Once
|
||||
using __cccl_exec_once_flag = void*;
|
||||
# define _LIBCUDACXX_EXEC_ONCE_INITIALIZER 0
|
||||
|
||||
// Thread ID
|
||||
using __cccl_thread_id = long;
|
||||
|
||||
// Thread
|
||||
# define _LIBCUDACXX_NULL_THREAD 0U
|
||||
|
||||
using __cccl_thread_t = void*;
|
||||
|
||||
// Thread Local Storage
|
||||
using __cccl_tls_key = long;
|
||||
|
||||
# define _LIBCUDACXX_TLS_DESTRUCTOR_CC __stdcall
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline void __cccl_thread_yield()
|
||||
{
|
||||
SwitchToThread();
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline void __cccl_thread_sleep_for(chrono::nanoseconds __ns)
|
||||
{
|
||||
using namespace chrono;
|
||||
// round-up to the nearest millisecond
|
||||
milliseconds __ms = duration_cast<milliseconds>(__ns + chrono::nanoseconds(999999));
|
||||
Sleep(static_cast<DWORD>(__ms.count()));
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_THREAD_API_WIN32
|
||||
|
||||
#endif // _CUDA_STD___THREAD_THREADING_SUPPORT_H
|
||||
Reference in New Issue
Block a user