[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
EngineX CI
171
cccl_upstream/libcudacxx/include/cuda/__event/event.h
Normal file
171
cccl_upstream/libcudacxx/include/cuda/__event/event.h
Normal file
@@ -0,0 +1,171 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA___EVENT_EVENT_H
|
||||
#define _CUDA___EVENT_EVENT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
# include <cuda/__device/device_ref.h>
|
||||
# include <cuda/__driver/driver_api.h>
|
||||
# include <cuda/__event/event_ref.h>
|
||||
# include <cuda/__runtime/ensure_current_context.h>
|
||||
# include <cuda/__utility/no_init.h>
|
||||
# include <cuda/std/__utility/to_underlying.h>
|
||||
# include <cuda/std/cstddef>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA
|
||||
|
||||
class timed_event;
|
||||
|
||||
//! @brief Flags to use when creating the event.
|
||||
enum class event_flags : unsigned
|
||||
{
|
||||
none = cudaEventDefault,
|
||||
blocking_sync = cudaEventBlockingSync,
|
||||
interprocess = cudaEventInterprocess,
|
||||
};
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_API constexpr event_flags operator|(event_flags __lhs, event_flags __rhs) noexcept
|
||||
{
|
||||
return static_cast<event_flags>(::cuda::std::to_underlying(__lhs) | ::cuda::std::to_underlying(__rhs));
|
||||
}
|
||||
|
||||
//! @brief An owning wrapper for an untimed `cudaEvent_t`.
|
||||
class event : public event_ref
|
||||
{
|
||||
friend class timed_event;
|
||||
|
||||
public:
|
||||
//! @brief Construct a new `event` object with timing disabled, and record
|
||||
//! the event in the specified stream.
|
||||
//!
|
||||
//! @throws cuda_error if the event creation fails.
|
||||
_CCCL_HOST_API explicit event(stream_ref __stream, event_flags __flags = event_flags::none);
|
||||
|
||||
//! @brief Construct a new `event` object with timing disabled. The event can only be recorded on streams from the
|
||||
//! specified device.
|
||||
//!
|
||||
//! @throws cuda_error if the event creation fails.
|
||||
_CCCL_HOST_API explicit event(device_ref __device, event_flags __flags = event_flags::none)
|
||||
: event(__device, ::cuda::std::to_underlying(__flags) | cudaEventDisableTiming)
|
||||
{}
|
||||
|
||||
//! @brief Construct a new `event` object into the moved-from state.
|
||||
//!
|
||||
//! @post `get()` returns `cudaEvent_t()`.
|
||||
_CCCL_HOST_API explicit constexpr event(no_init_t) noexcept
|
||||
: event_ref(::cudaEvent_t{})
|
||||
{}
|
||||
|
||||
//! @brief Move-construct a new `event` object
|
||||
//!
|
||||
//! @param __other
|
||||
//!
|
||||
//! @post `__other` is in a moved-from state.
|
||||
_CCCL_HOST_API constexpr event(event&& __other) noexcept
|
||||
: event_ref(::cuda::std::exchange(__other.__event_, {}))
|
||||
{}
|
||||
|
||||
// Disallow copy construction.
|
||||
event(const event&) = delete;
|
||||
|
||||
//! @brief Destroy the `event` object
|
||||
//!
|
||||
//! @note If the event fails to be destroyed, the error is silently ignored.
|
||||
_CCCL_HOST_API ~event()
|
||||
{
|
||||
if (__event_ != nullptr)
|
||||
{
|
||||
// Needs to call driver API in case current device is not set, runtime version would set dev 0 current
|
||||
// Alternative would be to store the device and push/pop here
|
||||
[[maybe_unused]] auto __status = ::cuda::__driver::__eventDestroyNoThrow(__event_);
|
||||
}
|
||||
}
|
||||
|
||||
//! @brief Move-assign an `event` object
|
||||
//!
|
||||
//! @param __other
|
||||
//!
|
||||
//! @post `__other` is in a moved-from state.
|
||||
_CCCL_HOST_API event& operator=(event&& __other) noexcept
|
||||
{
|
||||
event __tmp(::cuda::std::move(__other));
|
||||
::cuda::std::swap(__event_, __tmp.__event_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
// Disallow copy assignment.
|
||||
event& operator=(const event&) = delete;
|
||||
|
||||
//! @brief Construct an `event` object from a native `cudaEvent_t` handle.
|
||||
//!
|
||||
//! @param __evnt The native handle
|
||||
//!
|
||||
//! @return event The constructed `event` object
|
||||
//!
|
||||
//! @note The constructed `event` object takes ownership of the native handle.
|
||||
[[nodiscard]] static _CCCL_HOST_API event from_native_handle(::cudaEvent_t __evnt) noexcept
|
||||
{
|
||||
return event(__evnt);
|
||||
}
|
||||
|
||||
// Disallow construction from an `int`, e.g., `0`.
|
||||
static event from_native_handle(int) = delete;
|
||||
|
||||
// Disallow construction from `nullptr`.
|
||||
static event from_native_handle(::cuda::std::nullptr_t) = delete;
|
||||
|
||||
//! @brief Retrieve the native `cudaEvent_t` handle and give up ownership.
|
||||
//!
|
||||
//! @return cudaEvent_t The native handle being held by the `event` object.
|
||||
//!
|
||||
//! @post The event object is in a moved-from state.
|
||||
[[nodiscard]] _CCCL_HOST_API constexpr ::cudaEvent_t release() noexcept
|
||||
{
|
||||
return ::cuda::std::exchange(__event_, {});
|
||||
}
|
||||
|
||||
private:
|
||||
// Use `event::from_native_handle(e)` to construct an owning `event`
|
||||
// object from a `cudaEvent_t` handle.
|
||||
_CCCL_HOST_API explicit constexpr event(::cudaEvent_t __evnt) noexcept
|
||||
: event_ref(__evnt)
|
||||
{}
|
||||
|
||||
_CCCL_HOST_API explicit event(stream_ref __stream, unsigned __flags);
|
||||
|
||||
_CCCL_HOST_API explicit event(device_ref __device, unsigned __flags)
|
||||
: event_ref(::cudaEvent_t{})
|
||||
{
|
||||
[[maybe_unused]] __ensure_current_context __ctx_setter(__device);
|
||||
__event_ = ::cuda::__driver::__eventCreate(static_cast<unsigned>(__flags));
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
#endif // _CUDA___EVENT_EVENT_H
|
||||
158
cccl_upstream/libcudacxx/include/cuda/__event/event_ref.h
Normal file
158
cccl_upstream/libcudacxx/include/cuda/__event/event_ref.h
Normal file
@@ -0,0 +1,158 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA___EVENT_EVENT_REF_H
|
||||
#define _CUDA___EVENT_EVENT_REF_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
# include <cuda/__driver/driver_api.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/cassert>
|
||||
# include <cuda/std/cstddef>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA
|
||||
|
||||
class event;
|
||||
class timed_event;
|
||||
class stream_ref;
|
||||
|
||||
//! @brief An non-owning wrapper for an untimed `cudaEvent_t`.
|
||||
class event_ref
|
||||
{
|
||||
private:
|
||||
friend class event;
|
||||
friend class timed_event;
|
||||
|
||||
::cudaEvent_t __event_{};
|
||||
|
||||
public:
|
||||
using value_type = ::cudaEvent_t;
|
||||
|
||||
//! @brief Construct a new `event_ref` object from a `cudaEvent_t`
|
||||
//!
|
||||
//! This constructor provides an implicit conversion from `cudaEvent_t`
|
||||
//!
|
||||
//! @post `get() == __evnt`
|
||||
//!
|
||||
//! @note: It is the callers responsibility to ensure the `event_ref` does not
|
||||
//! outlive the event denoted by the `cudaEvent_t` handle.
|
||||
_CCCL_HOST_API constexpr event_ref(::cudaEvent_t __evnt) noexcept
|
||||
: __event_(__evnt)
|
||||
{}
|
||||
|
||||
/// Disallow construction from an `int`, e.g., `0`.
|
||||
event_ref(int) = delete;
|
||||
|
||||
/// Disallow construction from `nullptr`.
|
||||
event_ref(::cuda::std::nullptr_t) = delete;
|
||||
|
||||
//! @brief Records an event on the specified stream
|
||||
//!
|
||||
//! @param __stream
|
||||
//!
|
||||
//! @throws cuda_error if the event record fails
|
||||
_CCCL_HOST_API void record(stream_ref __stream) const;
|
||||
|
||||
//! @brief Synchronizes the event
|
||||
//!
|
||||
//! @throws cuda_error if waiting for the event fails
|
||||
_CCCL_HOST_API void sync() const
|
||||
{
|
||||
_CCCL_ASSERT(__event_ != nullptr, "cuda::event_ref::sync no event set");
|
||||
::cuda::__driver::__eventSynchronize(__event_);
|
||||
}
|
||||
|
||||
//! @brief Checks if all the work in the stream prior to the record of the event has completed.
|
||||
//!
|
||||
//! If is_done returns true, calling sync() on this event will return immediately
|
||||
//!
|
||||
//! @throws cuda_error if the event query fails
|
||||
[[nodiscard]] _CCCL_HOST_API bool is_done() const
|
||||
{
|
||||
_CCCL_ASSERT(__event_ != nullptr, "cuda::event_ref::sync no event set");
|
||||
::cudaError_t __status = ::cuda::__driver::__eventQueryNoThrow(__event_);
|
||||
if (__status == ::cudaSuccess)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
else if (__status == ::cudaErrorNotReady)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_THROW(::cuda::cuda_error, __status, "Failed to query CUDA event");
|
||||
}
|
||||
}
|
||||
|
||||
//! @brief Retrieve the native `cudaEvent_t` handle.
|
||||
//!
|
||||
//! @return cudaEvent_t The native handle being held by the event_ref object.
|
||||
[[nodiscard]] _CCCL_HOST_API constexpr ::cudaEvent_t get() const noexcept
|
||||
{
|
||||
return __event_;
|
||||
}
|
||||
|
||||
//! @brief Checks if the `event_ref` is valid
|
||||
//!
|
||||
//! @return true if the `event_ref` is valid, false otherwise.
|
||||
[[nodiscard]] _CCCL_HOST_API explicit constexpr operator bool() const noexcept
|
||||
{
|
||||
return __event_ != nullptr;
|
||||
}
|
||||
|
||||
//! @brief Compares two `event_ref`s for equality
|
||||
//!
|
||||
//! @note Allows comparison with `cudaEvent_t` due to implicit conversion to
|
||||
//! `event_ref`.
|
||||
//!
|
||||
//! @param __lhs The first `event_ref` to compare
|
||||
//! @param __rhs The second `event_ref` to compare
|
||||
//! @return true if `lhs` and `rhs` refer to the same `cudaEvent_t` object.
|
||||
[[nodiscard]] friend _CCCL_HOST_API constexpr bool operator==(event_ref __lhs, event_ref __rhs) noexcept
|
||||
{
|
||||
return __lhs.__event_ == __rhs.__event_;
|
||||
}
|
||||
|
||||
//! @brief Compares two `event_ref`s for inequality
|
||||
//!
|
||||
//! @note Allows comparison with `cudaEvent_t` due to implicit conversion to
|
||||
//! `event_ref`.
|
||||
//!
|
||||
//! @param __lhs The first `event_ref` to compare
|
||||
//! @param __rhs The second `event_ref` to compare
|
||||
//! @return true if `lhs` and `rhs` refer to different `cudaEvent_t` objects.
|
||||
[[nodiscard]] friend _CCCL_HOST_API constexpr bool operator!=(event_ref __lhs, event_ref __rhs) noexcept
|
||||
{
|
||||
return __lhs.__event_ != __rhs.__event_;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
#endif // _CUDA___EVENT_EVENT_REF_H
|
||||
117
cccl_upstream/libcudacxx/include/cuda/__event/timed_event.h
Normal file
117
cccl_upstream/libcudacxx/include/cuda/__event/timed_event.h
Normal file
@@ -0,0 +1,117 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA___EVENT_TIMED_EVENT_H
|
||||
#define _CUDA___EVENT_TIMED_EVENT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
# include <cuda/__device/device_ref.h>
|
||||
# include <cuda/__driver/driver_api.h>
|
||||
# include <cuda/__event/event.h>
|
||||
# include <cuda/__utility/no_init.h>
|
||||
# include <cuda/std/__chrono/duration.h>
|
||||
# include <cuda/std/__utility/to_underlying.h>
|
||||
# include <cuda/std/cstddef>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA
|
||||
|
||||
//! @brief An owning wrapper for a `cudaEvent_t` with timing enabled.
|
||||
class timed_event : public event
|
||||
{
|
||||
public:
|
||||
//! @brief Construct a new `timed_event` object with the specified flags
|
||||
//! and record the event on the specified stream.
|
||||
//!
|
||||
//! @throws cuda_error if the event creation fails.
|
||||
_CCCL_HOST_API explicit timed_event(stream_ref __stream, event_flags __flags = event_flags::none);
|
||||
|
||||
//! @brief Construct a new `timed_event` object with the specified flags. The event can only be recorded on streams
|
||||
//! from the specified device.
|
||||
//!
|
||||
//! @throws cuda_error if the event creation fails.
|
||||
_CCCL_HOST_API explicit timed_event(device_ref __device, event_flags __flags = event_flags::none)
|
||||
: event(__device, ::cuda::std::to_underlying(__flags))
|
||||
{}
|
||||
|
||||
//! @brief Construct a new `timed_event` object into the moved-from state.
|
||||
//!
|
||||
//! @post `get()` returns `cudaEvent_t()`.
|
||||
_CCCL_HOST_API explicit constexpr timed_event(no_init_t) noexcept
|
||||
: event(no_init)
|
||||
{}
|
||||
|
||||
timed_event(timed_event&&) noexcept = default;
|
||||
timed_event(const timed_event&) = delete;
|
||||
timed_event& operator=(timed_event&&) noexcept = default;
|
||||
timed_event& operator=(const timed_event&) = delete;
|
||||
|
||||
//! @brief Construct a `timed_event` object from a native `cudaEvent_t` handle.
|
||||
//!
|
||||
//! @param __evnt The native handle
|
||||
//!
|
||||
//! @return timed_event The constructed `timed_event` object
|
||||
//!
|
||||
//! @note The constructed `timed_event` object takes ownership of the native handle.
|
||||
[[nodiscard]] static _CCCL_HOST_API timed_event from_native_handle(::cudaEvent_t __evnt) noexcept
|
||||
{
|
||||
return timed_event(__evnt);
|
||||
}
|
||||
|
||||
// Disallow construction from an `int`, e.g., `0`.
|
||||
static timed_event from_native_handle(int) = delete;
|
||||
|
||||
// Disallow construction from `nullptr`.
|
||||
static timed_event from_native_handle(::cuda::std::nullptr_t) = delete;
|
||||
|
||||
//! @brief Compute the time elapsed between two `timed_event` objects.
|
||||
//!
|
||||
//! @throws cuda_error if the query for the elapsed time fails.
|
||||
//!
|
||||
//! @param __end The `timed_event` object representing the end time.
|
||||
//! @param __start The `timed_event` object representing the start time.
|
||||
//!
|
||||
//! @return cuda::std::chrono::nanoseconds The elapsed time in nanoseconds.
|
||||
//!
|
||||
//! @note The elapsed time has a resolution of approximately 0.5 microseconds.
|
||||
[[nodiscard]] friend _CCCL_HOST_API ::cuda::std::chrono::nanoseconds
|
||||
operator-(const timed_event& __end, const timed_event& __start)
|
||||
{
|
||||
const auto __ms = ::cuda::__driver::__eventElapsedTime(__start.get(), __end.get());
|
||||
return ::cuda::std::chrono::nanoseconds(static_cast<::cuda::std::chrono::nanoseconds::rep>(__ms * 1'000'000.0));
|
||||
}
|
||||
|
||||
private:
|
||||
// Use `timed_event::from_native_handle(e)` to construct an owning `timed_event`
|
||||
// object from a `cudaEvent_t` handle.
|
||||
_CCCL_HOST_API explicit constexpr timed_event(::cudaEvent_t __evnt) noexcept
|
||||
: event(__evnt)
|
||||
{}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
#endif // _CUDA___EVENT_TIMED_EVENT_H
|
||||
Reference in New Issue
Block a user