[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0
This commit is contained in:
EngineX CI
2026-07-30 09:35:51 +00:00
parent b4d01f481e
commit 56fd68e7dd
8871 changed files with 1454674 additions and 0 deletions

View File

@@ -0,0 +1,106 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_STD___THREAD_THREADING_SUPPORT_H
#define _CUDA_STD___THREAD_THREADING_SUPPORT_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__chrono/duration.h>
#include <cuda/std/__chrono/high_resolution_clock.h>
#if defined(_CCCL_HAS_THREAD_API_EXTERNAL)
# include <cuda/std/__thread/threading_support_external.h>
#endif // _CCCL_HAS_THREAD_API_EXTERNAL
#if defined(_CCCL_HAS_THREAD_API_CUDA)
# include <cuda/std/__thread/threading_support_cuda.h>
#elif defined(_CCCL_HAS_THREAD_API_PTHREAD)
# include <cuda/std/__thread/threading_support_pthread.h>
#elif defined(_CCCL_HAS_THREAD_API_WIN32)
# include <cuda/std/__thread/threading_support_win32.h>
#else // ^^^ _CCCL_HAS_THREAD_API_WIN32 ^^^ / vvv Unknown Thread API vvv
# error "Unknown Thread API"
#endif // Unknown Thread API
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_STD
#define _LIBCUDACXX_POLLING_COUNT 16
#if _CCCL_HOST_ARCH(ARM64) && _CCCL_OS(LINUX)
# define __LIBCUDACXX_ASM_THREAD_YIELD (asm volatile("yield" :: :);)
#elif _CCCL_HOST_ARCH(X86_64) && _CCCL_OS(LINUX)
# define __LIBCUDACXX_ASM_THREAD_YIELD (asm volatile("pause" :: :);)
#else // ^^^ _CCCL_HOST_ARCH(X86_64) ^^^ / vvv ! _CCCL_HOST_ARCH(X86_64) vvv
# define __LIBCUDACXX_ASM_THREAD_YIELD (;)
#endif // ! _CCCL_HOST_ARCH(X86_64)
_CCCL_HOST_DEVICE_API inline void __cccl_thread_yield_processor()
{
NV_IF_TARGET(NV_IS_HOST, __LIBCUDACXX_ASM_THREAD_YIELD)
}
template <class _Fn>
_CCCL_HOST_DEVICE_API inline bool __cccl_thread_poll_with_backoff(
_Fn&& __f, ::cuda::std::chrono::nanoseconds __max = ::cuda::std::chrono::nanoseconds::zero())
{
::cuda::std::chrono::high_resolution_clock::time_point const __start =
::cuda::std::chrono::high_resolution_clock::now();
for (int __count = 0;;)
{
if (__f())
{
return true;
}
if (__count < _LIBCUDACXX_POLLING_COUNT)
{
if (__count > (_LIBCUDACXX_POLLING_COUNT >> 1))
{
::cuda::std::__cccl_thread_yield_processor();
}
__count += 1;
continue;
}
::cuda::std::chrono::high_resolution_clock::duration const __elapsed =
::cuda::std::chrono::high_resolution_clock::now() - __start;
if (__max != ::cuda::std::chrono::nanoseconds::zero() && __max < __elapsed)
{
return false;
}
::cuda::std::chrono::nanoseconds const __step = __elapsed / 4;
if (__step >= ::cuda::std::chrono::milliseconds(1))
{
::cuda::std::__cccl_thread_sleep_for(::cuda::std::chrono::milliseconds(1));
}
else if (__step >= ::cuda::std::chrono::microseconds(10))
{
::cuda::std::__cccl_thread_sleep_for(__step);
}
else
{
::cuda::std::__cccl_thread_yield();
}
}
}
_CCCL_END_NAMESPACE_CUDA_STD
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA_STD___THREAD_THREADING_SUPPORT_H

View File

@@ -0,0 +1,47 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_STD___THREAD_THREADING_SUPPORT_CUDA_H
#define _CUDA_STD___THREAD_THREADING_SUPPORT_CUDA_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if defined(_CCCL_HAS_THREAD_API_CUDA)
# include <cuda/std/__chrono/duration.h>
# include <cuda/std/climits>
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_STD
_CCCL_HOST_DEVICE_API inline void __cccl_thread_yield() {}
_CCCL_HOST_DEVICE_API inline void __cccl_thread_sleep_for(::cuda::std::chrono::nanoseconds __ns){
NV_IF_TARGET(NV_PROVIDES_SM_70, ({
auto const __step = __ns.count();
_CCCL_ASSERT(__step < numeric_limits<unsigned>::max(), "invalid nanoseconds count");
::__nanosleep((unsigned) __step);
}))}
_CCCL_END_NAMESPACE_CUDA_STD
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_THREAD_API_CUDA
#endif // _CUDA_STD___THREAD_THREADING_SUPPORT_CUDA_H

View File

@@ -0,0 +1,41 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_STD___THREAD_THREADING_SUPPORT_EXTERNAL_H
#define _CUDA_STD___THREAD_THREADING_SUPPORT_EXTERNAL_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if defined(_CCCL_HAS_THREAD_API_EXTERNAL)
# include <cuda/std/__chrono/duration.h>
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_STD
_CCCL_HOST_DEVICE_API inline void __cccl_thread_yield();
_CCCL_HOST_DEVICE_API inline void __cccl_thread_sleep_for(::cuda::std::chrono::nanoseconds __ns);
_CCCL_END_NAMESPACE_CUDA_STD
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_THREAD_API_EXTERNAL
#endif // _CUDA_STD___THREAD_THREADING_SUPPORT_EXTERNAL_H

View File

@@ -0,0 +1,143 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_STD___THREAD_THREADING_SUPPORT_PTHREAD_H
#define _CUDA_STD___THREAD_THREADING_SUPPORT_PTHREAD_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if defined(_CCCL_HAS_THREAD_API_PTHREAD)
# include <cuda/std/__chrono/duration.h>
# include <cuda/std/__utility/cmp.h>
# include <cuda/std/climits>
# include <cuda/std/ctime>
# include <errno.h>
# include <pthread.h>
# include <sched.h>
# include <semaphore.h>
# if defined(__linux__)
# include <unistd.h>
# include <linux/futex.h>
# include <sys/syscall.h>
# endif // __linux__
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_STD
// Mutex
using __cccl_mutex_t = pthread_mutex_t;
# define _LIBCUDACXX_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
using __cccl_recursive_mutex_t = pthread_mutex_t;
// Condition Variable
using __cccl_condvar_t = pthread_cond_t;
# define _LIBCUDACXX_CONDVAR_INITIALIZER PTHREAD_COND_INITIALIZER
// Semaphore
using __cccl_semaphore_t = sem_t;
# define _LIBCUDACXX_SEMAPHORE_MAX SEM_VALUE_MAX
// Execute once
using __cccl_exec_once_flag = pthread_once_t;
# define _LIBCUDACXX_EXEC_ONCE_INITIALIZER PTHREAD_ONCE_INIT
// Thread id
using __cccl_thread_id = pthread_t;
// Thread
# define _LIBCUDACXX_NULL_THREAD 0U
using __cccl_thread_t = pthread_t;
// Thread Local Storage
using __cccl_tls_key = pthread_key_t;
# define _LIBCUDACXX_TLS_DESTRUCTOR_CC
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr timespec __cccl_to_timespec(const ::cuda::std::chrono::nanoseconds& __ns)
{
constexpr auto __ts_sec_max = numeric_limits<time_t>::max();
timespec __ts{};
const auto __s = ::cuda::std::chrono::duration_cast<chrono::seconds>(__ns);
if (::cuda::std::cmp_less(__s.count(), __ts_sec_max))
{
__ts.tv_sec = static_cast<time_t>(__s.count());
__ts.tv_nsec = static_cast<decltype(__ts.tv_nsec)>((__ns - __s).count());
}
else
{
__ts.tv_sec = __ts_sec_max;
__ts.tv_nsec = 999'999'999;
}
return __ts;
}
// Semaphore
_CCCL_HOST_DEVICE_API inline bool __cccl_semaphore_init(__cccl_semaphore_t* __sem, int __init)
{
return sem_init(__sem, 0, __init) == 0;
}
_CCCL_HOST_DEVICE_API inline bool __cccl_semaphore_destroy(__cccl_semaphore_t* __sem)
{
return sem_destroy(__sem) == 0;
}
_CCCL_HOST_DEVICE_API inline bool __cccl_semaphore_post(__cccl_semaphore_t* __sem)
{
return sem_post(__sem) == 0;
}
_CCCL_HOST_DEVICE_API inline bool __cccl_semaphore_wait(__cccl_semaphore_t* __sem)
{
return sem_wait(__sem) == 0;
}
_CCCL_HOST_DEVICE_API inline bool
__cccl_semaphore_wait_timed(__cccl_semaphore_t* __sem, ::cuda::std::chrono::nanoseconds const& __ns)
{
const auto __ts = __cccl_to_timespec(__ns);
return sem_timedwait(__sem, &__ts) == 0;
}
_CCCL_HOST_DEVICE_API inline void __cccl_thread_yield()
{
sched_yield();
}
_CCCL_HOST_DEVICE_API inline void __cccl_thread_sleep_for(::cuda::std::chrono::nanoseconds __ns)
{
auto __ts = __cccl_to_timespec(__ns);
while (nanosleep(&__ts, &__ts) == -1 && errno == EINTR)
;
}
_CCCL_END_NAMESPACE_CUDA_STD
# include <cuda/std/__cccl/epilogue.h>
#endif // !_CCCL_HAS_THREAD_API_PTHREAD
#endif // _CUDA_STD___THREAD_THREADING_SUPPORT_PTHREAD_H

View File

@@ -0,0 +1,87 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_STD___THREAD_THREADING_SUPPORT_WIN32_H
#define _CUDA_STD___THREAD_THREADING_SUPPORT_WIN32_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if defined(_CCCL_HAS_THREAD_API_WIN32)
# include <cuda/std/__chrono/duration.h>
# include <process.h>
# include <windows.h>
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_STD
// Mutex
using __cccl_mutex_t = void*;
# define _LIBCUDACXX_MUTEX_INITIALIZER 0
# if _CCCL_HOST_ARCH(ARM64) || _CCCL_HOST_ARCH(X86_64)
using __cccl_recursive_mutex_t = void* [5];
# else
# error Unsupported architecture
# endif
// Condition Variable
using __cccl_condvar_t = void*;
# define _LIBCUDACXX_CONDVAR_INITIALIZER 0
// Semaphore
using __cccl_semaphore_t = void*;
// Execute Once
using __cccl_exec_once_flag = void*;
# define _LIBCUDACXX_EXEC_ONCE_INITIALIZER 0
// Thread ID
using __cccl_thread_id = long;
// Thread
# define _LIBCUDACXX_NULL_THREAD 0U
using __cccl_thread_t = void*;
// Thread Local Storage
using __cccl_tls_key = long;
# define _LIBCUDACXX_TLS_DESTRUCTOR_CC __stdcall
_CCCL_HOST_DEVICE_API inline void __cccl_thread_yield()
{
SwitchToThread();
}
_CCCL_HOST_DEVICE_API inline void __cccl_thread_sleep_for(chrono::nanoseconds __ns)
{
using namespace chrono;
// round-up to the nearest millisecond
milliseconds __ms = duration_cast<milliseconds>(__ns + chrono::nanoseconds(999999));
Sleep(static_cast<DWORD>(__ms.count()));
}
_CCCL_END_NAMESPACE_CUDA_STD
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_THREAD_API_WIN32
#endif // _CUDA_STD___THREAD_THREADING_SUPPORT_H