[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA___RUNTIME_API_WRAPPER_H
|
||||
#define _CUDA___RUNTIME_API_WRAPPER_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__exception/cuda_error.h>
|
||||
#include <cuda/std/__exception/exception_macros.h>
|
||||
|
||||
#define _CCCL_TRY_CUDA_API(_NAME, _MSG, ...) \
|
||||
do \
|
||||
{ \
|
||||
const ::cudaError_t __status = _NAME(__VA_ARGS__); \
|
||||
if (__status != ::cudaSuccess) \
|
||||
{ \
|
||||
::cudaGetLastError(); /* clear CUDA error state */ \
|
||||
_CCCL_THROW(::cuda::cuda_error, __status, _MSG, #_NAME); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define _CCCL_ASSERT_CUDA_API(_NAME, _MSG, ...) \
|
||||
do \
|
||||
{ \
|
||||
[[maybe_unused]] const ::cudaError_t __status = _NAME(__VA_ARGS__); \
|
||||
::cudaGetLastError(); /* clear CUDA error state */ \
|
||||
_CCCL_ASSERT(__status == cudaSuccess, _MSG); \
|
||||
} while (0)
|
||||
|
||||
#define _CCCL_LOG_CUDA_API(_NAME, _MSG, ...) \
|
||||
[&]() { \
|
||||
const ::cudaError_t __status = _NAME(__VA_ARGS__); \
|
||||
if (__status != ::cudaSuccess) \
|
||||
{ \
|
||||
::cuda::__msg_storage __msg_buffer; \
|
||||
::cuda::__detail::__format_cuda_error(__msg_buffer, __status, _MSG, #_NAME); \
|
||||
::fprintf(stderr, "%s\n", __msg_buffer.__buffer); \
|
||||
::fflush(stderr); \
|
||||
} \
|
||||
::cudaGetLastError(); /* clear CUDA error state */ \
|
||||
return __status; \
|
||||
}()
|
||||
|
||||
#endif //_CUDA___RUNTIME_API_WRAPPER_H
|
||||
@@ -0,0 +1,99 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA___RUNTIME_ENSURE_CURRENT_CONTEXT_H
|
||||
#define _CUDA___RUNTIME_ENSURE_CURRENT_CONTEXT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
# include <cuda/__device/device_ref.h>
|
||||
# include <cuda/__device/physical_device.h>
|
||||
# include <cuda/__driver/driver_api.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
# ifndef _CCCL_DOXYGEN_INVOKED // Do not document
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA
|
||||
|
||||
class stream_ref;
|
||||
|
||||
//! @brief RAII helper which on construction sets the current context to the specified one.
|
||||
//! It sets the state back on destruction.
|
||||
//!
|
||||
struct [[maybe_unused]] __ensure_current_context
|
||||
{
|
||||
//! @brief Construct a new `__ensure_current_context` object and switch to the primary context of the specified
|
||||
//! device.
|
||||
//!
|
||||
//! @param new_device The device to switch the context to
|
||||
//!
|
||||
//! @throws cuda_error if the context switch fails
|
||||
_CCCL_HOST_API explicit __ensure_current_context(device_ref __new_device)
|
||||
{
|
||||
auto __ctx = ::cuda::__physical_devices()[__new_device.get()].__primary_context();
|
||||
::cuda::__driver::__ctxPush(__ctx);
|
||||
}
|
||||
|
||||
//! @brief Construct a new `__ensure_current_context` object and switch to the specified
|
||||
//! context.
|
||||
//!
|
||||
//! @param ctx The context to switch to
|
||||
//!
|
||||
//! @throws cuda_error if the context switch fails
|
||||
_CCCL_HOST_API explicit __ensure_current_context(::CUcontext __ctx)
|
||||
{
|
||||
::cuda::__driver::__ctxPush(__ctx);
|
||||
}
|
||||
|
||||
//! @brief Construct a new `__ensure_current_context` object and switch to the context
|
||||
//! under which the specified stream was created.
|
||||
//!
|
||||
//! @param stream Stream indicating the context to switch to
|
||||
//!
|
||||
//! @throws cuda_error if the context switch fails
|
||||
_CCCL_HOST_API explicit __ensure_current_context(stream_ref __stream);
|
||||
|
||||
__ensure_current_context(__ensure_current_context&&) = delete;
|
||||
__ensure_current_context(__ensure_current_context const&) = delete;
|
||||
__ensure_current_context& operator=(__ensure_current_context&&) = delete;
|
||||
__ensure_current_context& operator=(__ensure_current_context const&) = delete;
|
||||
|
||||
//! @brief Destroy the `__ensure_current_context` object and switch back to the original
|
||||
//! context.
|
||||
//!
|
||||
//! @throws cuda_error if the device switch fails. If the destructor is called
|
||||
//! during stack unwinding, the program is automatically terminated.
|
||||
_CCCL_HOST_API ~__ensure_current_context() noexcept(false)
|
||||
{
|
||||
// TODO would it make sense to assert here that we pushed and popped the same thing?
|
||||
::cuda::__driver::__ctxPop();
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA
|
||||
|
||||
# endif // _CCCL_DOXYGEN_INVOKED
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
#endif // _CUDA___RUNTIME_ENSURE_CURRENT_CONTEXT_H
|
||||
47
cccl_upstream/libcudacxx/include/cuda/__runtime/types.h
Normal file
47
cccl_upstream/libcudacxx/include/cuda/__runtime/types.h
Normal file
@@ -0,0 +1,47 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA___RUNTIME_TYPES_H
|
||||
#define _CUDA___RUNTIME_TYPES_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wmissing-braces")
|
||||
// clang complains about missing braces in CUmemLocation constructor but GCC complains if we add them
|
||||
|
||||
using memory_location = ::cudaMemLocation;
|
||||
# if _CCCL_CTK_AT_LEAST(12, 2)
|
||||
inline constexpr memory_location host_memory_location = {::cudaMemLocationTypeHost, 0};
|
||||
# endif // _CCCL_CTK_AT_LEAST(12, 2)
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA
|
||||
|
||||
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // __CUDA___RUNTIME_TYPES_H
|
||||
Reference in New Issue
Block a user