[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
EngineX CI
@@ -0,0 +1,99 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___EXCEPTION_CUDA_ERROR_H
|
||||
#define _CUDA_STD___EXCEPTION_CUDA_ERROR_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__exception/exception_macros.h>
|
||||
#include <cuda/std/__exception/msg_storage.h>
|
||||
#include <cuda/std/__host_stdlib/cstdio>
|
||||
#include <cuda/std/__host_stdlib/stdexcept>
|
||||
#include <cuda/std/source_location>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA
|
||||
|
||||
#if _CCCL_HAS_CTK()
|
||||
using __cuda_error_t = ::cudaError_t;
|
||||
#else
|
||||
using __cuda_error_t = int;
|
||||
#endif
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
namespace __detail
|
||||
{
|
||||
[[nodiscard]] _CCCL_HOST_API inline char* __format_cuda_error(
|
||||
::cuda::__msg_storage& __msg_buffer,
|
||||
const int __status,
|
||||
const char* __msg,
|
||||
const char* __api = nullptr,
|
||||
::cuda::std::source_location __loc = ::cuda::std::source_location::current()) noexcept
|
||||
{
|
||||
::snprintf(
|
||||
__msg_buffer.__buffer,
|
||||
512,
|
||||
"%s:%d %s%s%s(%d): %s",
|
||||
__loc.file_name(),
|
||||
__loc.line(),
|
||||
__api ? __api : "",
|
||||
__api ? " " : "",
|
||||
# if _CCCL_HAS_CTK()
|
||||
::cudaGetErrorString(::cudaError_t(__status)),
|
||||
# else // ^^^ _CCCL_HAS_CTK() ^^^ / vvv !_CCCL_HAS_CTK() vvv
|
||||
"cudaError",
|
||||
# endif // ^^^ !_CCCL_HAS_CTK() ^^^
|
||||
__status,
|
||||
__msg);
|
||||
return __msg_buffer.__buffer;
|
||||
}
|
||||
} // namespace __detail
|
||||
|
||||
/**
|
||||
* @brief Exception thrown when a CUDA error is encountered.
|
||||
*/
|
||||
class cuda_error : public ::std::runtime_error
|
||||
{
|
||||
public:
|
||||
_CCCL_HOST_API cuda_error(
|
||||
const __cuda_error_t __status,
|
||||
const char* __msg,
|
||||
const char* __api = nullptr,
|
||||
::cuda::std::source_location __loc = ::cuda::std::source_location::current(),
|
||||
__msg_storage __msg_buffer = {}) noexcept
|
||||
: ::std::runtime_error(::cuda::__detail::__format_cuda_error(__msg_buffer, __status, __msg, __api, __loc))
|
||||
, __status_(__status)
|
||||
{}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_API constexpr auto status() const noexcept -> __cuda_error_t
|
||||
{
|
||||
return __status_;
|
||||
}
|
||||
|
||||
private:
|
||||
__cuda_error_t __status_;
|
||||
};
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_STD___EXCEPTION_CUDA_ERROR_H
|
||||
@@ -0,0 +1,126 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___EXCEPTION_EXCEPTION_MACROS_H
|
||||
#define _CUDA_STD___EXCEPTION_EXCEPTION_MACROS_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__exception/terminate.h>
|
||||
#include <cuda/std/__host_stdlib/cstdio>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
struct __cccl_catch_any_lvalue
|
||||
{
|
||||
template <class _Tp>
|
||||
_CCCL_API operator _Tp&() const noexcept;
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
// The following macros are used to conditionally compile exception handling code. They
|
||||
// are used in the same way as `try` and `catch`, but they allow for different behavior
|
||||
// based on whether exceptions are enabled or not, and whether the code is being compiled
|
||||
// for device or not.
|
||||
//
|
||||
// Usage:
|
||||
// _CCCL_TRY
|
||||
// {
|
||||
// can_throw(); // Code that may throw an exception
|
||||
// }
|
||||
// _CCCL_CATCH (cuda_error& e) // Handle CUDA exceptions
|
||||
// {
|
||||
// printf("CUDA error: %s\n", e.what());
|
||||
// }
|
||||
// _CCCL_CATCH_ALL // Handle any other exceptions
|
||||
// {
|
||||
// printf("unknown error\n");
|
||||
// }
|
||||
//
|
||||
// Notes:
|
||||
// - the catch clause must always bind to a named variable
|
||||
|
||||
// Expand to keywords only for host code when exceptions are enabled. nvc++ in CUDA mode traps when an exception is
|
||||
// thrown in device code.
|
||||
#if _CCCL_HAS_EXCEPTIONS() && _CCCL_HOST_COMPILATION()
|
||||
# define _CCCL_TRY try
|
||||
# define _CCCL_CATCH catch
|
||||
# define _CCCL_CATCH_ALL catch (...)
|
||||
# define _CCCL_CATCH_FALLTHROUGH
|
||||
|
||||
// Even though nvc++ in CUDA mode replaces `throw` by `__trap()` call in device code, it instantiates the exception type
|
||||
// which can introduce some host only symbols to the nvvm ir (for example snprintf). So we need to wrap it by the
|
||||
// NV_IF_ELSE_TARGET macro.
|
||||
# define _CCCL_THROW(_TYPE, ...) \
|
||||
do \
|
||||
{ \
|
||||
NV_IF_ELSE_TARGET(NV_IS_HOST, (throw _TYPE(__VA_ARGS__);), (::cuda::std::terminate();)) \
|
||||
} while (0)
|
||||
# define _CCCL_RETHROW throw
|
||||
#else // ^^^ use exceptions ^^^ / vvv no exceptions vvv
|
||||
# define _CCCL_TRY \
|
||||
if constexpr (true) \
|
||||
{
|
||||
# define _CCCL_CATCH(...) \
|
||||
} \
|
||||
else if constexpr (false) \
|
||||
{ \
|
||||
for (__VA_ARGS__ = ::cuda::std::__cccl_catch_any_lvalue{}; false;)
|
||||
# define _CCCL_CATCH_ALL \
|
||||
} \
|
||||
else
|
||||
# define _CCCL_CATCH_FALLTHROUGH \
|
||||
} \
|
||||
else \
|
||||
{ \
|
||||
}
|
||||
|
||||
# if _CCCL_HOSTJIT()
|
||||
# define _CCCL_THROW(_TYPE, ...) \
|
||||
do \
|
||||
{ \
|
||||
_CCCL_ASSERT(false, "An instance of class " #_TYPE " would be thrown."); \
|
||||
::cuda::std::terminate(); \
|
||||
} while (0)
|
||||
# else // ^^^ _CCCL_HOSTJIT() ^^^ / vvv !_CCCL_HOSTJIT() vvv
|
||||
# define _CCCL_THROW(_TYPE, ...) \
|
||||
do \
|
||||
{ \
|
||||
NV_IF_ELSE_TARGET(NV_IS_HOST, \
|
||||
({ \
|
||||
::fprintf(stderr, \
|
||||
"%s:%u: An instance of class %s would be thrown.\n what(): %s\nAborted\n", \
|
||||
__FILE__, \
|
||||
__LINE__, \
|
||||
#_TYPE, \
|
||||
(_TYPE(__VA_ARGS__)).what()); \
|
||||
::fflush(stderr); \
|
||||
}), \
|
||||
({ _CCCL_ASSERT(false, "An instance of class " #_TYPE " would be thrown."); })) \
|
||||
::cuda::std::terminate(); \
|
||||
} while (0)
|
||||
# endif // !_CCCL_HOSTJIT()
|
||||
# define _CCCL_RETHROW ::cuda::std::terminate()
|
||||
#endif // ^^^ no exceptions ^^^
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_STD___EXCEPTION_EXCEPTION_MACROS_H
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___EXCEPTION_MSG_STORAGE_H
|
||||
#define _CUDA_STD___EXCEPTION_MSG_STORAGE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__cstddef/types.h>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA
|
||||
|
||||
struct __msg_storage
|
||||
{
|
||||
static constexpr ::cuda::std::size_t __size = 512;
|
||||
char __buffer[__size]{0};
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_STD___EXCEPTION_MSG_STORAGE_H
|
||||
@@ -0,0 +1,82 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___EXCEPTION_TERMINATE_H
|
||||
#define _CUDA_STD___EXCEPTION_TERMINATE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_TILE_COMPILATION()
|
||||
# include <cuda/std/cassert>
|
||||
#endif // !_CCCL_TILE_COMPILATION()
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
# include <stdlib.h>
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(4702) // unreachable code
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_NOVERSION // purposefully not using versioning namespace
|
||||
|
||||
[[noreturn]] _CCCL_API inline void __cccl_terminate() noexcept
|
||||
{
|
||||
#if _CCCL_TILE_COMPILATION()
|
||||
NV_IF_ELSE_TARGET(NV_IS_HOST, (::exit(-1);), (assert(false);))
|
||||
#else // ^^^ _CCCL_TILE_COMPILATION() ^^^ / vvv !_CCCL_TILE_COMPILATION()
|
||||
NV_IF_ELSE_TARGET(NV_IS_HOST, (::exit(-1);), (::__trap();))
|
||||
#endif // !_CCCL_TILE_COMPILATION()
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
#if 0 // Expose once atomic is universally available
|
||||
|
||||
using terminate_handler = void (*)();
|
||||
|
||||
# ifdef __CUDA_ARCH__
|
||||
__device__
|
||||
# endif // __CUDA_ARCH__
|
||||
static _CCCL_CONSTINIT ::cuda::std::atomic<terminate_handler>
|
||||
__cccl_terminate_handler{&__cccl_terminate};
|
||||
|
||||
_CCCL_API inline terminate_handler set_terminate(terminate_handler __func) noexcept
|
||||
{
|
||||
return __cccl_terminate_handler.exchange(__func);
|
||||
}
|
||||
_CCCL_API inline terminate_handler get_terminate() noexcept
|
||||
{
|
||||
return __cccl_terminate_handler.load(__func);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
[[noreturn]] _CCCL_API inline void terminate() noexcept
|
||||
{
|
||||
__cccl_terminate();
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_NOVERSION
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_STD___EXCEPTION_TERMINATE_H
|
||||
Reference in New Issue
Block a user