dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

[CCCL] 瘦身 + 补全: 移除 cudax/python/libcudacxx-tests 冗余文件, 新增 c2h 测试助手 + cmake 构建系统 + 8 个 CUDA thrust examples

Browse Source 
变更摘要:
- 删除: cudax/ (783 files, 7.2M) — 实验性组件，竞赛不需要
- 删除: python/ (226 files, 2.0M) — Python 绑定，竞赛不需要
- 删除: libcudacxx/{test,benchmarks,codegen,cmake,share} (4432 files, 31M)
  保留: libcudacxx/include/ (1463 headers, cuda::std 编译依赖)
- 新增: c2h/ (27 files) — CUB Catch2 测试辅助头文件，编译 243 个测试必需
- 新增: cmake/ (29 files) — CCCL 原生 CMake 构建系统
- 新增: thrust/examples/cuda/ (7 files) + cpp_integration/ (1 file)
  async_reduce, custom_temporary_allocation, explicit_cuda_stream,
  global_device_vector, range_view, unwrap_pointer, wrap_pointer, device

结果: cccl_upstream 从 74M→35M (瘦身 53%), 核心内容 100% 保留:
  27/27 tuning headers, 78 benchmarks, 243 tests,
  60 thrust examples, 18 CUB examples, 全部编译头文件
This commit is contained in:
muh-bot
2026-08-03 12:39:26 +00:00
parent a2a5dd8f00
commit 24ef6a91b5
5439 changed files with 0 additions and 719516 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
cccl_upstream/cudax/include/cuda/experimental/__graph/child_graph.cuh
View File

@@ -1,92 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDAX__GRAPH_CHILD_GRAPH_CUH
#define _CUDAX__GRAPH_CHILD_GRAPH_CUH
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_CTK_AT_LEAST(12, 2)
# include <cuda/experimental/__driver/driver_api.cuh>
# include <cuda/experimental/__graph/graph_builder.cuh>
# include <cuda/experimental/__graph/graph_builder_ref.cuh>
# include <cuda/experimental/__graph/graph_node_ref.cuh>
# include <cuda/experimental/__graph/path_builder.cuh>
# include <cuda/std/__cccl/prologue.h>
namespace cuda::experimental
{
//! \brief Adds a child graph node to a CUDA graph path.
//!
//! The entire subgraph described by \p __child is embedded as a single node in the parent
//! graph. All nodes in the child graph execute before any successor of the new child-graph
//! node.
//!
//! \param __pb Path builder to insert the node into.
//! \param __child A `graph_builder_ref` whose underlying graph will become the child.
//! \return A `graph_node_ref` for the newly added child-graph node.
//! \throws cuda::std::cuda_error if node creation fails.
_CCCL_HOST_API inline graph_node_ref insert_child_graph(path_builder& __pb, graph_builder_ref __child)
{
auto __deps = __pb.get_dependencies();
::CUgraphNodeParams __params{};
__params.type = ::CU_GRAPH_NODE_TYPE_GRAPH;
__params.graph.graph = __child.get();
auto __node = ::cuda::experimental::__driver::__graphAddNode(
__pb.get_native_graph_handle(), __deps.data(), __deps.size(), &__params);
__pb.__clear_and_set_dependency_node(__node);
return graph_node_ref{__node, __pb.get_native_graph_handle()};
}
# if _CCCL_CTK_AT_LEAST(12, 9)
//! \brief Adds a child graph node to a CUDA graph path, transferring ownership.
//!
//! The child graph is moved into the parent graph node. After this call, the
//! `graph_builder` is left in a null state and the parent graph owns the child's
//! lifetime.
//!
//! \param __pb Path builder to insert the node into.
//! \param __child An rvalue `graph_builder` whose graph will be moved into the parent.
//! \return A `graph_node_ref` for the newly added child-graph node.
//! \throws cuda::std::cuda_error if node creation fails.
_CCCL_HOST_API inline graph_node_ref insert_child_graph(path_builder& __pb, graph_builder&& __child)
{
auto __deps = __pb.get_dependencies();
::CUgraphNodeParams __params{};
__params.type = ::CU_GRAPH_NODE_TYPE_GRAPH;
__params.graph.graph = __child.get();
__params.graph.ownership = ::CU_GRAPH_CHILD_GRAPH_OWNERSHIP_MOVE;
auto __node = ::cuda::experimental::__driver::__graphAddNode(
__pb.get_native_graph_handle(), __deps.data(), __deps.size(), &__params);
(void) __child.release();
__pb.__clear_and_set_dependency_node(__node);
return graph_node_ref{__node, __pb.get_native_graph_handle()};
}
# endif // _CCCL_CTK_AT_LEAST(12, 9)
} // namespace cuda::experimental
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_CTK_AT_LEAST(12, 2)
#endif // _CUDAX__GRAPH_CHILD_GRAPH_CUH

48
cccl_upstream/cudax/include/cuda/experimental/__graph/concepts.cuh
View File

@@ -1,48 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDAX_GRAPH_CONCEPTS
#define __CUDAX_GRAPH_CONCEPTS
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__type_traits/disjunction.h>
#include <cuda/std/__type_traits/is_same.h>
#include <cuda/experimental/__graph/fwd.cuh>
#include <cuda/std/__cccl/prologue.h>
namespace cuda::experimental
{
// Concept to check if T is a graph dependency or contains them (either path_builder or graph_node_ref)
// TODO we might do something more abstract here rather than just checking specific types
template <typename T>
_CCCL_CONCEPT graph_dependency =
::cuda::std::is_same_v<::cuda::std::decay_t<T>, path_builder>
|| ::cuda::std::is_same_v<::cuda::std::decay_t<T>, graph_node_ref>;
// Concept to check if T can insert nodes into a graph
// TODO we might do something more abstract here rather than just checking specific types
template <typename T>
_CCCL_CONCEPT graph_inserter = ::cuda::std::is_same_v<::cuda::std::decay_t<T>, path_builder>;
} // namespace cuda::experimental
#include <cuda/std/__cccl/epilogue.h>
#endif // __CUDAX_GRAPH_CONCEPTS

187
cccl_upstream/cudax/include/cuda/experimental/__graph/conditional_node.cuh
View File

@@ -1,187 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDAX__GRAPH_CONDITIONAL_NODE_CUH
#define _CUDAX__GRAPH_CONDITIONAL_NODE_CUH
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_CTK_AT_LEAST(12, 4)
# include <cuda/experimental/__driver/driver_api.cuh>
# include <cuda/experimental/__graph/graph_builder_ref.cuh>
# include <cuda/experimental/__graph/graph_node_ref.cuh>
# include <cuda/experimental/__graph/path_builder.cuh>
# include <cuda/std/__cccl/prologue.h>
namespace cuda::experimental
{
//! \brief A thin, non-owning wrapper around a `cudaGraphConditionalHandle`.
//!
//! A conditional handle is a graph-scoped token whose value at runtime controls whether
//! the body of an `if` or `while` conditional node executes. The handle is owned by
//! the graph — there is no destroy API — so this wrapper is trivially copyable and
//! safe to pass by value into device kernels.
//!
//! Users can construct a handle directly, or let `make_if_node` / `make_while_node`
//! create one automatically.
//!
//! \rst
//! .. _cudax-graph-conditional-handle:
//! \endrst
struct conditional_handle
{
//! \brief Creates a conditional handle for the given graph.
//!
//! \param __graph Graph in which the conditional node will be inserted.
//! \param __default_val Initial value of the handle (true = execute body, false = skip).
//! \throws cuda::std::cuda_error if `cudaGraphConditionalHandleCreate` fails.
_CCCL_HOST_API explicit conditional_handle(graph_builder_ref __graph, bool __default_val = true)
: __handle_(::cuda::experimental::__driver::__graphConditionalHandleCreate(
__graph.get(), __graph.get_device().__primary_context(), __default_val, ::cudaGraphCondAssignDefault))
{}
//! \brief Sets the runtime value of the conditional handle from device code.
//!
//! \param __value Non-zero to execute the body, zero to skip.
_CCCL_DEVICE void set_value(bool __value) const noexcept
{
::cudaGraphSetConditional(__handle_, __value);
}
//! \brief Convenience: enables execution of the conditional body (sets the handle to 1).
_CCCL_DEVICE void enable() const noexcept
{
set_value(true);
}
//! \brief Convenience: disables execution of the conditional body (sets the handle to 0).
_CCCL_DEVICE void disable() const noexcept
{
set_value(false);
}
//! \brief Returns the underlying `cudaGraphConditionalHandle`.
[[nodiscard]] _CCCL_NODEBUG_HOST_API ::cudaGraphConditionalHandle get() const noexcept
{
return __handle_;
}
private:
::cudaGraphConditionalHandle __handle_{};
};
//! \brief Result of adding a conditional node.
//!
//! Contains the newly created conditional node, the body graph that should be
//! populated by the caller, and the conditional handle to pass into body kernels.
struct conditional_node_result
{
graph_node_ref node; //!< The conditional node in the parent graph.
graph_builder_ref body_graph; //!< The body graph to populate with operations.
conditional_handle handle; //!< The handle to control execution from device code.
};
_CCCL_HOST_API inline conditional_node_result
__make_conditional_node(path_builder& __pb, conditional_handle __handle, ::CUgraphConditionalNodeType __type)
{
auto __deps = __pb.get_dependencies();
::CUgraphNodeParams __params{};
__params.type = ::CU_GRAPH_NODE_TYPE_CONDITIONAL;
__params.conditional.handle = __handle.get();
__params.conditional.type = __type;
__params.conditional.size = 1;
__params.conditional.ctx = __pb.get_device().__primary_context();
auto __node = ::cuda::experimental::__driver::__graphAddNode(
__pb.get_native_graph_handle(), __deps.data(), __deps.size(), &__params);
__pb.__clear_and_set_dependency_node(__node);
return {graph_node_ref{__node, __pb.get_native_graph_handle()},
graph_builder_ref{__params.conditional.phGraph_out[0], __pb.get_device()},
__handle};
}
//! \brief Adds an `if`-conditional node to a CUDA graph path.
//!
//! At runtime, if the value of the handle is non-zero the body graph executes once;
//! otherwise it is skipped entirely.
//!
//! The caller must populate the returned `body_graph` with all operations that should
//! run conditionally before the parent graph is instantiated.
//!
//! \param __pb Path builder to insert the node into.
//! \param __default_val Initial handle value (true = execute, false = skip). Ignored when
//! \p __handle is provided.
//! \return A `conditional_node_result` containing the node ref, body graph, and handle.
//! \throws cuda::std::cuda_error if node creation fails.
_CCCL_HOST_API inline conditional_node_result make_if_node(path_builder& __pb, bool __default_val = true)
{
conditional_handle __handle{__pb.get_graph(), __default_val};
return __make_conditional_node(__pb, __handle, ::CU_GRAPH_COND_TYPE_IF);
}
//! \brief Adds an `if`-conditional node reusing an existing conditional handle.
//!
//! \param __pb Path builder to insert the node into.
//! \param __handle An existing conditional handle (e.g. shared with another node).
//! \return A `conditional_node_result` containing the node ref, body graph, and handle.
//! \throws cuda::std::cuda_error if node creation fails.
_CCCL_HOST_API inline conditional_node_result make_if_node(path_builder& __pb, conditional_handle __handle)
{
return __make_conditional_node(__pb, __handle, ::CU_GRAPH_COND_TYPE_IF);
}
//! \brief Adds a `while`-conditional node to a CUDA graph path.
//!
//! At runtime, the body graph is executed repeatedly as long as the handle value
//! is non-zero at the start of each iteration (including the first).
//!
//! The caller must populate the returned `body_graph` before instantiating the parent
//! graph. The body is responsible for calling `handle.set_value(false)` or `handle.disable()`
//! to terminate the loop.
//!
//! \param __pb Path builder to insert the node into.
//! \param __default_val Initial handle value (true = enter loop, false = skip).
//! \return A `conditional_node_result` containing the node ref, body graph, and handle.
//! \throws cuda::std::cuda_error if node creation fails.
_CCCL_HOST_API inline conditional_node_result make_while_node(path_builder& __pb, bool __default_val = true)
{
conditional_handle __handle{__pb.get_graph(), __default_val};
return __make_conditional_node(__pb, __handle, ::CU_GRAPH_COND_TYPE_WHILE);
}
//! \brief Adds a `while`-conditional node reusing an existing conditional handle.
//!
//! \param __pb Path builder to insert the node into.
//! \param __handle An existing conditional handle (e.g. shared with another node).
//! \return A `conditional_node_result` containing the node ref, body graph, and handle.
//! \throws cuda::std::cuda_error if node creation fails.
_CCCL_HOST_API inline conditional_node_result make_while_node(path_builder& __pb, conditional_handle __handle)
{
return __make_conditional_node(__pb, __handle, ::CU_GRAPH_COND_TYPE_WHILE);
}
} // namespace cuda::experimental
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_CTK_AT_LEAST(12, 4)
#endif // _CUDAX__GRAPH_CONDITIONAL_NODE_CUH

147
cccl_upstream/cudax/include/cuda/experimental/__graph/copy_bytes.cuh
View File

@@ -1,147 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDAX__GRAPH_COPY_BYTES_CUH
#define _CUDAX__GRAPH_COPY_BYTES_CUH
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_CTK_AT_LEAST(12, 2)
# include <cuda/__algorithm/common.h>
# include <cuda/__stream/launch_transform.h>
# include <cuda/__stream/stream_ref.h>
# include <cuda/__type_traits/is_trivially_copyable.h>
# include <cuda/std/__concepts/concept_macros.h>
# include <cuda/std/__exception/exception_macros.h>
# include <cuda/std/__host_stdlib/stdexcept>
# include <cuda/std/__type_traits/is_const.h>
# include <cuda/std/cstddef>
# include <cuda/std/span>
# include <cuda/experimental/__driver/driver_api.cuh>
# include <cuda/experimental/__graph/graph_node_ref.cuh>
# include <cuda/experimental/__graph/path_builder.cuh>
# include <cuda/std/__cccl/prologue.h>
namespace cuda::experimental
{
template <typename _SrcTy, typename _DstTy>
_CCCL_HOST_API graph_node_ref
__copy_bytes_graph_impl(path_builder& __pb, ::cuda::std::span<_SrcTy> __src, ::cuda::std::span<_DstTy> __dst)
{
static_assert(!::cuda::std::is_const_v<_DstTy>, "Copy destination can't be const");
static_assert(::cuda::is_trivially_copyable_v<_SrcTy> && ::cuda::is_trivially_copyable_v<_DstTy>,
"Copy source and destination element types must be trivially copyable");
if (__src.size_bytes() > __dst.size_bytes())
{
_CCCL_THROW(::std::invalid_argument, "Copy destination is too small to fit the source data");
}
if (__src.size_bytes() == 0)
{
return graph_node_ref{};
}
auto __deps = __pb.get_dependencies();
::CUgraphNodeParams __params{};
__params.type = ::CU_GRAPH_NODE_TYPE_MEMCPY;
__params.memcpy.copyCtx = __pb.get_device().__primary_context();
__params.memcpy.copyParams.srcMemoryType = ::CU_MEMORYTYPE_UNIFIED;
__params.memcpy.copyParams.srcDevice = reinterpret_cast<::CUdeviceptr>(__src.data());
__params.memcpy.copyParams.dstMemoryType = ::CU_MEMORYTYPE_UNIFIED;
__params.memcpy.copyParams.dstDevice = reinterpret_cast<::CUdeviceptr>(__dst.data());
__params.memcpy.copyParams.WidthInBytes = __src.size_bytes();
__params.memcpy.copyParams.Height = 1;
__params.memcpy.copyParams.Depth = 1;
auto __node = ::cuda::experimental::__driver::__graphAddNode(
__pb.get_native_graph_handle(), __deps.data(), __deps.size(), &__params);
__pb.__clear_and_set_dependency_node(__node);
return graph_node_ref{__node, __pb.get_native_graph_handle()};
}
template <typename _SrcElem,
typename _SrcExtents,
typename _SrcLayout,
typename _SrcAccessor,
typename _DstElem,
typename _DstExtents,
typename _DstLayout,
typename _DstAccessor>
_CCCL_HOST_API graph_node_ref __copy_bytes_graph_impl(
path_builder& __pb,
::cuda::std::mdspan<_SrcElem, _SrcExtents, _SrcLayout, _SrcAccessor> __src,
::cuda::std::mdspan<_DstElem, _DstExtents, _DstLayout, _DstAccessor> __dst)
{
static_assert(::cuda::std::is_constructible_v<_DstExtents, _SrcExtents>,
"Multidimensional copy requires both source and destination extents to be compatible");
static_assert(::cuda::std::is_same_v<_SrcLayout, _DstLayout>,
"Multidimensional copy requires both source and destination layouts to match");
if (!__dst.is_exhaustive())
{
_CCCL_THROW(::std::invalid_argument, "copy_bytes supports only exhaustive mdspans");
}
if (__src.extents() != __dst.extents())
{
_CCCL_THROW(::std::invalid_argument, "Copy destination size differs from the source");
}
return __copy_bytes_graph_impl(
__pb,
::cuda::std::span(__src.data_handle(), __src.mapping().required_span_size()),
::cuda::std::span(__dst.data_handle(), __dst.mapping().required_span_size()));
}
//! \brief Adds a memcpy node to a CUDA graph path that copies bytes from source to destination.
_CCCL_TEMPLATE(typename _SrcTy, typename _DstTy)
_CCCL_REQUIRES(::cuda::__spannable<::cuda::transformed_device_argument_t<_SrcTy>>
_CCCL_AND ::cuda::__spannable<::cuda::transformed_device_argument_t<_DstTy>>)
_CCCL_HOST_API graph_node_ref copy_bytes(path_builder& __pb, _SrcTy&& __src, _DstTy&& __dst)
{
return __copy_bytes_graph_impl(
__pb,
::cuda::std::span(
::cuda::launch_transform(::cuda::stream_ref{::cuda::invalid_stream}, ::cuda::std::forward<_SrcTy>(__src))),
::cuda::std::span(
::cuda::launch_transform(::cuda::stream_ref{::cuda::invalid_stream}, ::cuda::std::forward<_DstTy>(__dst))));
}
//! \brief Adds a memcpy node for mdspan source and destination.
_CCCL_TEMPLATE(typename _SrcTy, typename _DstTy)
_CCCL_REQUIRES(::cuda::__mdspannable<::cuda::transformed_device_argument_t<_SrcTy>>
_CCCL_AND ::cuda::__mdspannable<::cuda::transformed_device_argument_t<_DstTy>>)
_CCCL_HOST_API graph_node_ref copy_bytes(path_builder& __pb, _SrcTy&& __src, _DstTy&& __dst)
{
return __copy_bytes_graph_impl(
__pb,
::cuda::__as_mdspan(
::cuda::launch_transform(::cuda::stream_ref{::cuda::invalid_stream}, ::cuda::std::forward<_SrcTy>(__src))),
::cuda::__as_mdspan(
::cuda::launch_transform(::cuda::stream_ref{::cuda::invalid_stream}, ::cuda::std::forward<_DstTy>(__dst))));
}
} // namespace cuda::experimental
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_CTK_AT_LEAST(12, 2)
#endif // _CUDAX__GRAPH_COPY_BYTES_CUH

59
cccl_upstream/cudax/include/cuda/experimental/__graph/depends_on.cuh
View File

@@ -1,59 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDAX_GRAPH_DEPENDS_ON
#define __CUDAX_GRAPH_DEPENDS_ON
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/array>
#include <cuda/experimental/__graph/fwd.cuh>
#include <cuda/experimental/__graph/graph_node_ref.cuh>
#include <cuda_runtime_api.h>
#include <cuda/std/__cccl/prologue.h>
namespace cuda::experimental
{
//! \brief Builds an array of graph nodes that represent dependencies. It is for use as a
//! parameter to the `graph_builder::add` function.
//!
//! \tparam _Nodes Variadic template parameter representing the types of the graph nodes.
//! Each type must be either `graph_node_ref` or `cudaGraphNode_t`.
//! \param __nodes The graph nodes to add as dependencies to a new node.
//! \return A object of type `cuda::std::array<cudaGraphNode_t, sizeof...(_Nodes)>`
//! containing the references to the provided graph nodes.
//!
//! \note A static assertion ensures that all provided arguments are convertible to
//! `graph_node_ref`. If this condition is not met, a compilation error will occur.
// TODO graph_node_ref needs a graph argument if this function would accept cudaGraphNode_t
// TODO we should consider defining a type that also wraps a device and a graph and making it a graph_inserter,
// and then we could return it here. It would serve as a non-advancing alternative to path_builder.
template <class... _Nodes>
_CCCL_NODEBUG_HOST_API constexpr auto depends_on(const _Nodes&... __nodes) noexcept
-> ::cuda::std::array<cudaGraphNode_t, sizeof...(_Nodes)>
{
return ::cuda::std::array<cudaGraphNode_t, sizeof...(_Nodes)>{{graph_node_ref(__nodes).get()...}};
}
} // namespace cuda::experimental
#include <cuda/std/__cccl/epilogue.h>
#endif // __CUDAX_GRAPH_DEPENDS_ON

128
cccl_upstream/cudax/include/cuda/experimental/__graph/fill_bytes.cuh
View File

@@ -1,128 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDAX__GRAPH_FILL_BYTES_CUH
#define _CUDAX__GRAPH_FILL_BYTES_CUH
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_CTK_AT_LEAST(12, 2)
# include <cuda/__algorithm/common.h>
# include <cuda/__stream/launch_transform.h>
# include <cuda/__stream/stream_ref.h>
# include <cuda/__type_traits/is_trivially_copyable.h>
# include <cuda/std/__concepts/concept_macros.h>
# include <cuda/std/__exception/exception_macros.h>
# include <cuda/std/__host_stdlib/stdexcept>
# include <cuda/std/__type_traits/is_const.h>
# include <cuda/std/cstdint>
# include <cuda/std/span>
# include <cuda/experimental/__driver/driver_api.cuh>
# include <cuda/experimental/__graph/concepts.cuh>
# include <cuda/experimental/__graph/graph_node_ref.cuh>
# include <cuda/experimental/__graph/path_builder.cuh>
# include <cuda/std/__cccl/prologue.h>
namespace cuda::experimental
{
template <typename _DstTy, ::cuda::std::size_t _DstSize>
_CCCL_HOST_API graph_node_ref
__fill_bytes_graph_impl(path_builder& __pb, ::cuda::std::span<_DstTy, _DstSize> __dst, ::cuda::std::uint8_t __value)
{
static_assert(!::cuda::std::is_const_v<_DstTy>, "Fill destination can't be const");
static_assert(::cuda::is_trivially_copyable_v<_DstTy>, "Fill destination element type must be trivially copyable");
auto __deps = __pb.get_dependencies();
::CUgraphNodeParams __params{};
__params.type = ::CU_GRAPH_NODE_TYPE_MEMSET;
__params.memset.dst = reinterpret_cast<::CUdeviceptr>(__dst.data());
__params.memset.pitch = __dst.size_bytes();
__params.memset.value = __value;
__params.memset.elementSize = 1;
__params.memset.width = __dst.size_bytes();
__params.memset.height = 1;
__params.memset.ctx = __pb.get_device().__primary_context();
auto __node = ::cuda::experimental::__driver::__graphAddNode(
__pb.get_native_graph_handle(), __deps.data(), __deps.size(), &__params);
__pb.__clear_and_set_dependency_node(__node);
return graph_node_ref{__node, __pb.get_native_graph_handle()};
}
template <typename _DstElem, typename _DstExtents, typename _DstLayout, typename _DstAccessor>
_CCCL_HOST_API graph_node_ref __fill_bytes_graph_impl(
path_builder& __pb,
::cuda::std::mdspan<_DstElem, _DstExtents, _DstLayout, _DstAccessor> __dst,
::cuda::std::uint8_t __value)
{
if (!__dst.is_exhaustive())
{
_CCCL_THROW(::std::invalid_argument, "fill_bytes supports only exhaustive mdspans");
}
return __fill_bytes_graph_impl(
__pb, ::cuda::std::span(__dst.data_handle(), __dst.mapping().required_span_size()), __value);
}
//! \brief Adds a memset node to a CUDA graph path that bytewise-fills the destination.
//!
//! This overload is selected when the destination (after applying `launch_transform`) is
//! a contiguous range convertible to `cuda::std::span`. The element type must be trivially
//! copyable and non-const. The pointer captured in the node must remain valid until the
//! graph executes.
//!
//! \param __pb Path builder to insert the node into.
//! \param __dst Destination memory to fill.
//! \param __value Byte value to write to every byte of the destination.
//! \return A `graph_node_ref` for the newly added memset node.
//! \throws cuda::std::cuda_error if node creation fails.
_CCCL_TEMPLATE(typename _DstTy)
_CCCL_REQUIRES(::cuda::__spannable<::cuda::transformed_device_argument_t<_DstTy>>)
_CCCL_HOST_API graph_node_ref fill_bytes(path_builder& __pb, _DstTy&& __dst, ::cuda::std::uint8_t __value)
{
return __fill_bytes_graph_impl(
__pb,
::cuda::std::span(
::cuda::launch_transform(::cuda::stream_ref{::cuda::invalid_stream}, ::cuda::std::forward<_DstTy>(__dst))),
__value);
}
//! @overload
//! This overload is selected when the destination (after applying `launch_transform`) is
//! a `cuda::std::mdspan`. The mdspan must be exhaustive. The element type must be trivially
//! copyable and non-const. The pointer captured in the node must remain valid until the
//! graph executes.
_CCCL_TEMPLATE(typename _DstTy)
_CCCL_REQUIRES(::cuda::__mdspannable<::cuda::transformed_device_argument_t<_DstTy>>)
_CCCL_HOST_API graph_node_ref fill_bytes(path_builder& __pb, _DstTy&& __dst, ::cuda::std::uint8_t __value)
{
return __fill_bytes_graph_impl(
__pb,
::cuda::__as_mdspan(
::cuda::launch_transform(::cuda::stream_ref{::cuda::invalid_stream}, ::cuda::std::forward<_DstTy>(__dst))),
__value);
}
} // namespace cuda::experimental
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_CTK_AT_LEAST(12, 2)
#endif // _CUDAX__GRAPH_FILL_BYTES_CUH

44
cccl_upstream/cudax/include/cuda/experimental/__graph/fwd.cuh
View File

@@ -1,44 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDAX_GRAPH_FWD
#define __CUDAX_GRAPH_FWD
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/array>
#include <cuda_runtime_api.h>
#include <cuda/std/__cccl/prologue.h>
namespace cuda::experimental
{
struct graph_builder;
struct graph_node_ref;
struct graph;
struct path_builder;
template <class... _Nodes>
_CCCL_NODEBUG_HOST_API constexpr auto depends_on(const _Nodes&... __nodes) noexcept
-> ::cuda::std::array<cudaGraphNode_t, sizeof...(_Nodes)>;
} // namespace cuda::experimental
#include <cuda/std/__cccl/epilogue.h>
#endif // __CUDAX_GRAPH_FWD

155
cccl_upstream/cudax/include/cuda/experimental/__graph/graph.cuh
View File

@@ -1,155 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDAX_GRAPH_GRAPH
#define __CUDAX_GRAPH_GRAPH
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__runtime/api_wrapper.h>
#include <cuda/std/__utility/exchange.h>
#include <cuda/std/__utility/swap.h>
#include <cuda/experimental/__driver/driver_api.cuh>
#include <cuda/experimental/__stream/stream_ref.cuh>
#include <cuda_runtime_api.h>
#include <cuda/std/__cccl/prologue.h>
namespace cuda::experimental
{
//! \brief An owning wrapper for a cudaGraphExec_t handle
//!
//! The `graph` class provides a safe and convenient interface for managing
//! the lifecycle of a `cudaGraphExec_t` object, ensuring proper cleanup and
//! resource management. It supports move semantics, resource release, and
//! launch of the CUDA graph.
//!
//! \note The `graph` object is not directly constructible. One is obtained
//! by calling the `instantiate()` method on a `graph_builder` object.
//! \sa cuda::experimental::graph_builder
//!
//! \rst
//! .. _cudax-graph-graph:
//! \endrst
struct _CCCL_TYPE_VISIBILITY_DEFAULT graph
{
//! \brief Move constructor for `graph`.
//! \param __other The `graph` object to move from.
//! \note After the move, the source object is left in the empty state.
//! \post `__other.get() == nullptr`
_CCCL_HOST_API constexpr graph(graph&& __other) noexcept
: __exec_{::cuda::std::exchange(__other.__exec_, nullptr)}
{}
//! \brief Destructor for `graph`.
//! \details Ensures proper cleanup of the CUDA graph execution object.
//! \throws None
_CCCL_HOST_API _CCCL_CONSTEXPR_CXX20 ~graph()
{
reset();
}
//! \brief Move assignment operator for `graph`.
//! \param __other The `graph` object to move from.
//! \return A reference to the current object.
//! \note After the move, the source object is left in the empty state.
//! \throws None
//! \post `__other.get() == nullptr`
_CCCL_HOST_API constexpr auto operator=(graph&& __other) noexcept -> graph&
{
swap(__other);
__other.reset();
return *this;
}
//! \brief Swaps the contents of this `graph` with another.
//! \param __other The `graph` object to swap with.
//! \throws None
_CCCL_HOST_API constexpr void swap(graph& __other) noexcept
{
::cuda::std::swap(__exec_, __other.__exec_);
}
//! \brief Retrieves the underlying CUDA graph execution object.
//! \return The `cudaGraphExec_t` handle.
//! \throws None
[[nodiscard]] _CCCL_NODEBUG_HOST_API constexpr auto get() const noexcept -> cudaGraphExec_t
{
return __exec_;
}
//! \brief Releases ownership of the CUDA graph execution object.
//! \return The `cudaGraphExec_t` handle, leaving this object in a null state.
//! \throws None
//! \post `get() == nullptr`
[[nodiscard]] _CCCL_NODEBUG_HOST_API constexpr auto release() noexcept -> cudaGraphExec_t
{
return ::cuda::std::exchange(__exec_, nullptr);
}
//! \brief Resets the `graph` object, destroying the underlying CUDA graph execution object.
//! \throws cuda::std::cuda_error if `cudaGraphExecDestroy` fails.
//! \post `get() == nullptr`
_CCCL_HOST_API constexpr void reset() noexcept
{
if (auto __exec = ::cuda::std::exchange(__exec_, nullptr))
{
[[maybe_unused]] auto __status = ::cuda::experimental::__driver::__graphExecDestroyNoThrow(__exec);
_CCCL_ASSERT(__status == cudaSuccess, "cuGraphExecDestroy failed");
}
}
//! \brief Constructs a `graph` object from a native CUDA graph execution handle.
//! \param __exec The native CUDA graph execution handle to construct the `graph` object from.
//! \throws None
//! \post `get() == __exec`
[[nodiscard]] _CCCL_HOST_API static _CCCL_CONSTEXPR_CXX20 auto from_native_handle(cudaGraphExec_t __exec) noexcept
-> graph
{
return graph{__exec};
}
//! \brief Launches the CUDA graph execution object on the specified stream.
//! \param __stream The stream on which to launch the graph.
//! \throws cuda::std::cuda_error if `cudaGraphLaunch` fails.
_CCCL_HOST_API void launch(stream_ref __stream)
{
::cuda::experimental::__driver::__graphLaunch(__exec_, __stream.get());
}
private:
friend struct graph_builder_ref;
_CCCL_HIDE_FROM_ABI graph() = default;
//! \brief Constructs a `graph` object from a native CUDA graph execution handle.
//! \param __exec The native CUDA graph execution handle to construct the `graph` object from.
//! \throws None
_CCCL_HOST_API explicit constexpr graph(cudaGraphExec_t __exec) noexcept
: __exec_{__exec}
{}
cudaGraphExec_t __exec_ = nullptr; //!< The underlying CUDA graph execution handle.
};
} // namespace cuda::experimental
#include <cuda/std/__cccl/epilogue.h>
#endif // __CUDAX_GRAPH_GRAPH

198
cccl_upstream/cudax/include/cuda/experimental/__graph/graph_builder.cuh
View File

@@ -1,198 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDAX_GRAPH_GRAPH_BUILDER
#define __CUDAX_GRAPH_GRAPH_BUILDER
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__utility/exchange.h>
#include <cuda/experimental/__driver/driver_api.cuh>
#include <cuda/experimental/__graph/graph_builder_ref.cuh>
#include <cuda/std/__cccl/prologue.h>
namespace cuda::experimental
{
//! \brief An owning wrapper type for a cudaGraph_t handle
//!
//! The `graph_builder` class provides a high-level interface for creating, managing, and
//! manipulating CUDA graphs. It ensures proper resource management and simplifies the
//! process of working with CUDA graph APIs.
//!
//! Features:
//! - Supports construction, destruction, and copying of CUDA graphs.
//! - Provides methods for adding nodes and dependencies to the graph.
//! - Allows instantiation of the graph into an executable form.
//! - Ensures proper cleanup of CUDA resources.
//!
//! Usage:
//! - Create an instance of `graph_builder` to represent a CUDA graph.
//! - Use the `add` methods to add nodes and dependencies to the graph.
//! - Instantiate the graph using the `instantiate` method to obtain an executable graph.
//! - Use the `reset` method to release resources when the graph is no longer needed.
//!
//! Thread Safety:
//! - This class is not thread-safe. Concurrent access to the same `graph_builer` object
//! must be synchronized externally.
//!
//! Exception Safety:
//! - Methods that interact with CUDA APIs may throw ``cuda::std::cuda_error`` if the
//! underlying CUDA operation fails.
//! - Move operations leave the source object in a valid but unspecified state.
//!
//! \rst
//! .. _cudax-graph-graph-builder:
//! \endrst
struct _CCCL_TYPE_VISIBILITY_DEFAULT graph_builder : graph_builder_ref
{
//! \brief Constructs a new, empty CUDA graph.
//! \param __dev The device on which graph nodes will execute.
//! \throws cuda::std::cuda_error if `cudaGraphCreate` fails.
_CCCL_HOST_API explicit graph_builder(device_ref __dev)
: graph_builder_ref(nullptr, __dev)
{
__graph_ = ::cuda::experimental::__driver::__graphCreate();
}
//! \brief Constructs a new, empty CUDA graph.
//! \details The nodes in the graph will execute on the default device 0.
//! \throws cuda::std::cuda_error if `cudaGraphCreate` fails.
_CCCL_HOST_API explicit graph_builder()
: graph_builder(device_ref{0})
{}
/// Disallow construction from an `int`, e.g., `0`.
graph_builder(int) = delete;
/// Disallow construction from `nullptr`.
graph_builder(::cuda::std::nullptr_t) = delete;
//! \brief Constructs an uninitialized CUDA graph.
//! \param __dev The device on which graph nodes will execute, default to device 0.
//! \throws None
_CCCL_HOST_API explicit constexpr graph_builder(no_init_t, device_ref __dev = device_ref{0}) noexcept
: graph_builder_ref(nullptr, __dev)
{}
//! \brief Move constructor for `graph_builder`.
//! \param __other The `graph_builder` object to move from.
//! \note After the move, the source object is left in the empty state.
//! \throws None
//! \post `__other.get() == nullptr`
_CCCL_HOST_API constexpr graph_builder(graph_builder&& __other) noexcept
: graph_builder_ref(::cuda::std::exchange(__other.__graph_, nullptr), __other.__dev_)
{}
//! \brief Copy constructor for `graph_builder`.
//! \param __other The `graph_builder` object to copy from.
//! \throws cuda::std::cuda_error if `cudaGraphClone` fails.
//! \post `get() == __other.get()`
_CCCL_HOST_API constexpr graph_builder(graph_builder_ref __other)
: graph_builder_ref(nullptr, __other.__dev_)
{
if (__other.__graph_)
{
__graph_ = ::cuda::experimental::__driver::__graphClone(__other.__graph_);
}
}
//! \brief Destructor for `graph_builder`.
//! \details Ensures proper cleanup of the CUDA graph object.
//! \throws None
_CCCL_HOST_API _CCCL_CONSTEXPR_CXX20 ~graph_builder()
{
reset();
}
//! \brief Move assignment operator for `graph_builder`.
//! \param __other The `graph_builder` object to move from.
//! \return A reference to the current object.
//! \note After the move, the source object is left in the empty state.
//! \post `__other.get() == nullptr`
//! \throws None
_CCCL_HOST_API constexpr auto operator=(graph_builder&& __other) noexcept -> graph_builder&
{
if (this != &__other)
{
swap(__other);
__other.reset();
}
return *this;
}
//! \brief Copy assignment operator for `graph_builder`.
//! \param __other The `graph_builder` object to copy from.
//! \return A reference to the current object.
//! \post `get() == __other.get()`
//! \throws cuda::std::cuda_error if `cudaGraphClone` fails.
_CCCL_HOST_API _CCCL_CONSTEXPR_CXX20 auto operator=(graph_builder_ref __other) -> graph_builder&
{
if (this != &__other)
{
operator=(graph_builder(__other));
}
return *this;
}
//! \brief Releases ownership of the CUDA graph object.
//! \return The `cudaGraph_t` handle, leaving this object in a null state.
//! \throws None
//! \post `get() == nullptr`
[[nodiscard]] _CCCL_NODEBUG_HOST_API constexpr auto release() noexcept -> cudaGraph_t
{
return ::cuda::std::exchange(__graph_, nullptr);
}
//! \brief Resets the `graph_builder` object, destroying the underlying CUDA graph object.
//! \post `get() == nullptr`
_CCCL_HOST_API constexpr void reset() noexcept
{
if (auto __graph = ::cuda::std::exchange(__graph_, nullptr))
{
[[maybe_unused]] auto __status = ::cuda::experimental::__driver::__graphDestroyNoThrow(__graph);
_CCCL_ASSERT(__status == cudaSuccess, "cuGraphDestroy failed");
}
}
//! \brief Constructs a `graph_builder` object from a native CUDA graph handle.
//! \param __graph The native CUDA graph handle to construct the `graph_builder` object from.
//! \param __dev The device on which graph nodes will execute, default to device 0.
//! \throws None
//! \post `get() == __graph`
[[nodiscard]] _CCCL_HOST_API static _CCCL_CONSTEXPR_CXX20 auto
from_native_handle(cudaGraph_t __graph, device_ref __dev) noexcept -> graph_builder
{
return graph_builder{__graph, __dev};
}
private:
//! \brief Constructs a `graph_builder` object from a native CUDA graph handle.
//! \param __graph The native CUDA graph handle to construct the `graph_builder` object from.
//! \param __dev The device on which graph nodes will execute, default to device 0.
//! \throws None
_CCCL_HOST_API explicit constexpr graph_builder(cudaGraph_t __graph, device_ref __dev) noexcept
: graph_builder_ref(__graph, __dev)
{}
};
} // namespace cuda::experimental
#include <cuda/std/__cccl/epilogue.h>
#endif // __CUDAX_GRAPH_GRAPH_BUILDER

290
cccl_upstream/cudax/include/cuda/experimental/__graph/graph_builder_ref.cuh
View File

@@ -1,290 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDAX_GRAPH_GRAPH_BUILDER_REF
#define __CUDAX_GRAPH_GRAPH_BUILDER_REF
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__runtime/api_wrapper.h>
#include <cuda/__runtime/ensure_current_context.h>
#include <cuda/std/__type_traits/is_same.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/__utility/swap.h>
#include <cuda/std/array>
#include <cuda/std/cstddef>
#include <cuda/std/span>
#include <cuda/experimental/__driver/driver_api.cuh>
#include <cuda/experimental/__graph/graph.cuh>
#include <cuda/experimental/__graph/graph_node_ref.cuh>
#include <cuda_runtime_api.h>
#include <cuda/std/__cccl/prologue.h>
// work around breathe "_CUDAX_CONSTEXPR_FRIEND friend" bug.
// See: https://github.com/breathe-doc/breathe/issues/916
#if defined(_CCCL_DOXYGEN_INVOKED)
# define _CUDAX_CONSTEXPR_FRIEND friend
#else
# define _CUDAX_CONSTEXPR_FRIEND constexpr friend
#endif
namespace cuda::experimental
{
//! \brief An owning wrapper type for a cudaGraph_t handle
//!
//! The `graph_builder` class provides a high-level interface for creating, managing, and
//! manipulating CUDA graphs. It ensures proper resource management and simplifies the
//! process of working with CUDA graph APIs.
//!
//! Features:
//! - Supports construction, destruction, and copying of CUDA graphs.
//! - Provides methods for adding nodes and dependencies to the graph.
//! - Allows instantiation of the graph into an executable form.
//! - Ensures proper cleanup of CUDA resources.
//!
//! Usage:
//! - Create an instance of `graph_builder` to represent a CUDA graph.
//! - Use the `add` methods to add nodes and dependencies to the graph.
//! - Instantiate the graph using the `instantiate` method to obtain an executable graph.
//! - Use the `reset` method to release resources when the graph is no longer needed.
//!
//! Thread Safety:
//! - This class is not thread-safe. Concurrent access to the same `graph_builer` object
//! must be synchronized externally.
//!
//! Exception Safety:
//! - Methods that interact with CUDA APIs may throw ``cuda::std::cuda_error`` if the
//! underlying CUDA operation fails.
//! - Move operations leave the source object in a valid but unspecified state.
//!
//! \rst
//! .. _cudax-graph-graph-builder:
//! \endrst
struct _CCCL_TYPE_VISIBILITY_DEFAULT graph_builder_ref
{
//! \brief Constructs a new, empty CUDA graph.
//! \param __graph The CUDA graph handle to wrap.
//! \param __dev The device on which graph nodes will execute.
//! \throws cuda::std::cuda_error if `cudaGraphCreate` fails.
_CCCL_HOST_API constexpr graph_builder_ref(cudaGraph_t __graph, device_ref __dev) noexcept
: __dev_{__dev}
, __graph_{__graph}
{}
//! \brief Compares two `graph_builder` objects for equality.
//!
//! \param __lhs The left-hand side `graph_builder` object to compare.
//! \param __rhs The right-hand side `graph_builder` object to compare.
//! \return `true` if both `graph_builder` objects are equal, `false` otherwise.
[[nodiscard]] _CCCL_HOST_API _CUDAX_CONSTEXPR_FRIEND bool
operator==(const graph_builder_ref& __lhs, const graph_builder_ref& __rhs) noexcept
{
return __lhs.__graph_ == __rhs.__graph_;
}
//! \brief Compares two `graph_builder` objects for inequality.
//!
//! \param __lhs The left-hand side `graph_builder` object to compare.
//! \param __rhs The right-hand side `graph_builder` object to compare.
//! \return `true` if both `graph_builder` objects are not equal, `false` otherwise.
[[nodiscard]] _CCCL_HOST_API _CUDAX_CONSTEXPR_FRIEND bool
operator!=(const graph_builder_ref& __lhs, const graph_builder_ref& __rhs) noexcept
{
return !(__lhs == __rhs);
}
//! \brief Test whether a `graph_builder` object is null.
//! \return `true` if `__rhs` is null, `false` otherwise.
[[nodiscard]] _CCCL_HOST_API _CUDAX_CONSTEXPR_FRIEND bool
operator==(::cuda::std::nullptr_t, const graph_builder_ref& __rhs) noexcept
{
return !static_cast<bool>(__rhs);
}
//! \brief Test whether a `graph_builder` object is null.
//! \return `true` if `__rhs` is null, `false` otherwise.
[[nodiscard]] _CCCL_HOST_API _CUDAX_CONSTEXPR_FRIEND bool
operator==(const graph_builder_ref& __lhs, ::cuda::std::nullptr_t) noexcept
{
return !static_cast<bool>(__lhs);
}
//! \brief Test whether a `graph_builder` object is not null.
//! \return `true` if `__rhs` is not null, `false` otherwise.
[[nodiscard]] _CCCL_HOST_API _CUDAX_CONSTEXPR_FRIEND bool
operator!=(::cuda::std::nullptr_t, const graph_builder_ref& __rhs) noexcept
{
return static_cast<bool>(__rhs);
}
//! \brief Test whether a `graph_builder` object is not null.
//! \return `true` if `__lhs` is not null, `false` otherwise.
[[nodiscard]] _CCCL_HOST_API _CUDAX_CONSTEXPR_FRIEND bool
operator!=(const graph_builder_ref& __lhs, ::cuda::std::nullptr_t) noexcept
{
return static_cast<bool>(__lhs);
}
//! \brief Checks if the graph handle is valid.
//!
//! \details This operator allows the graph builder to be used in a
//! boolean context to determine if it is valid. A valid graph builder
//! is one where the internal node pointer is not `nullptr`.
//!
//! \return `true` if the internal node pointer is not `nullptr`, otherwise `false`.
[[nodiscard]] _CCCL_HOST_API explicit constexpr operator bool() const noexcept
{
return __graph_ != nullptr;
}
//! \brief Checks if the graph is not null.
//! \return `true` if the internal graph handle is null, otherwise `false`.
[[nodiscard]] _CCCL_HOST_API constexpr auto operator!() const noexcept -> bool
{
return !static_cast<bool>(*this);
}
//! \brief Swaps the contents of this `graph_builder` with another.
//! \param __other The `graph_builder` object to swap with.
//! \throws None
_CCCL_HOST_API constexpr void swap(graph_builder_ref& __other) noexcept
{
::cuda::std::swap(__graph_, __other.__graph_);
}
//! \brief Retrieves the underlying CUDA graph object.
//! \return The `cudaGraph_t` handle.
//! \throws None
[[nodiscard]] _CCCL_NODEBUG_HOST_API constexpr auto get() const noexcept -> cudaGraph_t
{
return __graph_;
}
//! \brief Retrieves the device on which the graph is built.
//! \return The device on which the graph is built.
[[nodiscard]] _CCCL_NODEBUG_HOST_API constexpr auto get_device() const noexcept -> device_ref
{
return __dev_;
}
//! \brief Adds a new root node to the graph.
//! \tparam _Node The type of the node to add.
//! \param __node The descriptor of the node to add to the graph.
//! \return A `graph_node_ref` representing the added node. The graph object owns the
//! new node.
//! \throws cuda::std::cuda_error if adding the node fails.
template <class _Node>
[[nodiscard]] _CCCL_NODEBUG_HOST_API constexpr auto add(_Node __node) -> graph_node_ref
{
return add(_CCCL_MOVE(__node), ::cuda::std::span<cudaGraphNode_t, 0>{});
}
//! \brief Adds a new node to the graph with specified dependencies.
//!
//! This function creates a new node in the graph and establishes dependencies
//! between the newly created node and the provided dependency nodes.
//!
//! \tparam _Node The type of the node to be added.
//! \tparam _Extent The extent of the span representing the dependencies.
//!
//! \param __node The descriptor of the node to be added to the graph.
//! \param __deps An array of `cudaGraphNode_t` handles representing the dependencies of
//! the new node. Each node in this span will become a dependency of the newly created
//! node.
//!
//! \return A `graph_node_ref` object representing the newly created node in the graph.
//! The graph object owns the new node.
//!
//! \throws cuda::std::cuda_error If the CUDA API call `cudaGraphAddDependencies` fails.
//!
//! \details
//! - The function first creates a new node in the graph using the provided `_Node` object.
//! - It initializes an array of "dependant" nodes, where all dependant nodes correspond
//! to the newly created node.
//! - The function then uses the CUDA API `cudaGraphAddDependencies` to establish the
//! dependencies between the newly created node and the nodes provided in the `__deps`
//! span.
//! - If the number of dependencies is small, a stack-allocated buffer is used;
//! otherwise, a dynamically allocated array is used to store the dependant nodes.
template <class _Node, size_t _Np>
_CCCL_HOST_API constexpr auto add(_Node __node, ::cuda::std::array<cudaGraphNode_t, _Np> __deps) -> graph_node_ref
{
return add(_CCCL_MOVE(__node), ::cuda::std::span{__deps});
}
//! \overload
template <class _Node, size_t _Extent>
_CCCL_HOST_API constexpr auto add(_Node __node, ::cuda::std::span<cudaGraphNode_t, _Extent> __deps) -> graph_node_ref
{
// assert that the node descriptor returns a graph_node_ref object:
static_assert(::cuda::std::_IsSame<decltype(__node.__add_to_graph(__graph_, __deps)), graph_node_ref>::value,
"node descriptors must return a graph_node_ref");
return __node.__add_to_graph(__graph_, __deps);
}
//! \brief Retrieves the number of nodes in the graph.
//! \return The number of nodes in the graph.
//! \throws cuda::std::cuda_error if `cudaGraphGetNodes` fails.
[[nodiscard]] _CCCL_HOST_API size_t node_count() const
{
return ::cuda::experimental::__driver::__graphGetNodeCount(__graph_);
}
//! \brief Instantiates the CUDA graph into a `graph_exec` object.
//! \return A `graph_exec` object representing the instantiated graph.
//! \throws cuda::std::cuda_error if `cudaGraphInstantiate` fails.
_CCCL_HOST_API auto instantiate() -> graph
{
_CCCL_ASSERT(__graph_ != nullptr, "cannot instantiate a NULL graph");
::cuda::__ensure_current_context __ctx_guard{__dev_};
return graph{::cuda::experimental::__driver::__graphInstantiate(__graph_)};
}
private:
friend struct graph_builder;
//! \brief Adds this graph as a child graph to the parent graph.
//! \param __parent The parent graph to which this graph will be added.
//! \return A `graph_node_ref` representing the added child graph.
//! \throws cuda::std::cuda_error if `cudaGraphAddChildGraphNode` fails.
#if _CCCL_CTK_AT_LEAST(12, 2)
template <size_t _Extent>
[[nodiscard]] _CCCL_HOST_API auto
__add_to_graph(cudaGraph_t __parent, ::cuda::std::span<cudaGraphNode_t, _Extent> __deps) -> graph_node_ref
{
::CUgraphNodeParams __params{};
__params.type = ::CU_GRAPH_NODE_TYPE_GRAPH;
__params.graph.graph = __graph_;
graph_node_ref __child;
__child.__graph_ = __graph_;
__child.__node_ = ::cuda::experimental::__driver::__graphAddNode(__parent, __deps.data(), __deps.size(), &__params);
return __child;
}
#endif // _CCCL_CTK_AT_LEAST(12, 2)
device_ref __dev_; //!< The device on which the graph is built.
cudaGraph_t __graph_ = nullptr; //!< The underlying CUDA graph handle.
};
} // namespace cuda::experimental
#include <cuda/std/__cccl/epilogue.h>
#endif // __CUDAX_GRAPH_GRAPH_BUILDER_REF

162
cccl_upstream/cudax/include/cuda/experimental/__graph/graph_memory_resource.cuh
View File

@@ -1,162 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDAX__GRAPH_GRAPH_MEMORY_RESOURCE_CUH
#define _CUDAX__GRAPH_GRAPH_MEMORY_RESOURCE_CUH
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_CTK_AT_LEAST(12, 2)
# include <cuda/__driver/driver_api.h>
# include <cuda/__memory_resource/properties.h>
# include <cuda/__stream/stream_ref.h>
# include <cuda/std/cstddef>
# include <cuda/experimental/__driver/driver_api.cuh>
# include <cuda/experimental/__graph/graph_node_ref.cuh>
# include <cuda/experimental/__graph/path_builder.cuh>
# include <cuda/std/__cccl/prologue.h>
namespace cuda::experimental
{
//! @brief A memory resource that allocates and frees device memory as CUDA graph nodes.
//!
//! Constructed with a device_ref that determines where memory is allocated.
//! Inserts `cuGraphAddMemAllocNode` and `cuGraphAddMemFreeNode` nodes into a graph
//! via a path_builder. Also supports stream-based deallocation for memory that outlives
//! the graph execution.
//!
struct graph_memory_resource
{
//! @brief Construct a graph memory resource for the specified device.
//! @param __dev The device on which memory will be allocated.
_CCCL_HOST_API explicit graph_memory_resource(device_ref __dev) noexcept
: __dev_(__dev)
{}
//! @brief Insert a memory allocation node into the graph.
//! @param __pb The path builder to insert the alloc node into.
//! @param __size Number of bytes to allocate.
//! @param __alignment Alignment requirement (unused by the CUDA graph alloc API, reserved).
//! @return Device pointer to the allocated memory.
_CCCL_HOST_API void* allocate(path_builder& __pb,
::cuda::std::size_t __size,
::cuda::std::size_t __alignment = cuda::mr::default_cuda_malloc_alignment)
{
(void) __alignment;
if (__size == 0)
{
return nullptr;
}
auto __deps = __pb.get_dependencies();
auto [__node, __dptr] = ::cuda::experimental::__driver::__graphAddMemAllocNode(
__pb.get_native_graph_handle(), __deps.data(), __deps.size(), __size, __dev_.get());
__pb.__clear_and_set_dependency_node(__node);
return reinterpret_cast<void*>(__dptr);
}
//! @brief Insert a memory free node into the graph.
//! @param __pb The path builder to insert the free node into.
//! @param __ptr Device pointer previously returned by allocate().
//! @param __size Number of bytes (unused, kept for interface symmetry).
//! @param __alignment Alignment (unused, kept for interface symmetry).
_CCCL_HOST_API void deallocate(
path_builder& __pb,
void* __ptr,
::cuda::std::size_t __size = 0,
::cuda::std::size_t __alignment = ::cuda::mr::default_cuda_malloc_alignment) noexcept
{
(void) __size;
(void) __alignment;
if (__ptr == nullptr)
{
return;
}
auto __deps = __pb.get_dependencies();
auto [__node, __err] = ::cuda::experimental::__driver::__graphAddMemFreeNodeNoThrow(
__pb.get_native_graph_handle(), __deps.data(), __deps.size(), reinterpret_cast<::CUdeviceptr>(__ptr));
_CCCL_ASSERT(__err == ::cudaSuccess, "Failed to add a memory free node to graph");
__pb.__clear_and_set_dependency_node(__node);
}
//! @brief Free device memory asynchronously on a stream.
//! @param __stream The stream on which to free the memory.
//! @param __ptr Device pointer previously returned by allocate().
//! @param __size Number of bytes (unused, kept for interface symmetry).
//! @param __alignment Alignment (unused, kept for interface symmetry).
_CCCL_HOST_API void deallocate(
::cuda::stream_ref __stream,
void* __ptr,
::cuda::std::size_t __size = 0,
::cuda::std::size_t __alignment = ::cuda::mr::default_cuda_malloc_alignment)
{
(void) __size;
(void) __alignment;
if (__ptr == nullptr)
{
return;
}
_CCCL_ASSERT_CUDA_API(
::cuda::__driver::__freeAsyncNoThrow,
"graph_memory_resource::deallocate failed",
reinterpret_cast<::CUdeviceptr>(__ptr),
__stream.get());
}
//! @brief Returns the device this resource allocates on.
[[nodiscard]] _CCCL_HOST_API device_ref device() const noexcept
{
return __dev_;
}
//! @brief Enables the \c device_accessible property.
_CCCL_HOST_API friend constexpr void get_property(graph_memory_resource const&, ::cuda::mr::device_accessible) noexcept
{}
using default_queries = ::cuda::mr::properties_list<::cuda::mr::device_accessible>;
_CCCL_HOST_API friend bool operator==(const graph_memory_resource& __lhs, const graph_memory_resource& __rhs) noexcept
{
return __lhs.__dev_ == __rhs.__dev_;
}
_CCCL_HOST_API friend bool operator!=(const graph_memory_resource& __lhs, const graph_memory_resource& __rhs) noexcept
{
return !(__lhs == __rhs);
}
private:
device_ref __dev_;
};
} // namespace cuda::experimental
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_CTK_AT_LEAST(12, 2)
#endif // _CUDAX__GRAPH_GRAPH_MEMORY_RESOURCE_CUH

280
cccl_upstream/cudax/include/cuda/experimental/__graph/graph_node_ref.cuh
View File

@@ -1,280 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDAX_GRAPH_GRAPH_NODE_REF
#define __CUDAX_GRAPH_GRAPH_NODE_REF
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__runtime/api_wrapper.h>
#include <cuda/std/__algorithm/fill.h>
#include <cuda/std/__memory/unique_ptr.h>
#include <cuda/std/__ranges/size.h>
#include <cuda/std/__utility/swap.h>
#include <cuda/std/cstddef>
#include <cuda/std/span>
#include <cuda/experimental/__driver/driver_api.cuh>
#include <cuda/experimental/__graph/fwd.cuh>
#include <cuda/experimental/__graph/graph_node_type.cuh>
#include <cuda_runtime_api.h>
#include <cuda/std/__cccl/prologue.h>
// work around breathe "_CUDAX_CONSTEXPR_FRIEND friend" bug.
// See: https://github.com/breathe-doc/breathe/issues/916
#if defined(_CCCL_DOXYGEN_INVOKED)
# define _CUDAX_CONSTEXPR_FRIEND friend
#else
# define _CUDAX_CONSTEXPR_FRIEND constexpr friend
#endif
namespace cuda::experimental
{
//! \brief A reference wrapper for a CUDA graph node.
//!
//! This structure provides an interface to manage and interact with a CUDA graph node
//! within a CUDA graph. It includes functionality for swapping, retrieving node information,
//! and managing dependencies between nodes.
//!
//! \rst
//! .. _cudax-graph-graph-node-ref:
//! \endrst
struct graph_node_ref
{
//! \brief Default constructor.
_CCCL_HIDE_FROM_ABI graph_node_ref() = default;
/// Disallow construction from an `int`, e.g., `0`.
graph_node_ref(int, int = 0) = delete;
/// Disallow construction from `nullptr`.
graph_node_ref(::cuda::std::nullptr_t, ::cuda::std::nullptr_t = nullptr) = delete;
//! \brief Constructs a graph_node_ref with a given CUDA graph node and graph.
//! \param __node The CUDA graph node.
//! \param __graph The CUDA graph containing the node.
//! \pre Both of __node and __graph are non-null.
//! \post `get() == __node`
_CCCL_NODEBUG_HOST_API explicit constexpr graph_node_ref(cudaGraphNode_t __node, cudaGraph_t __graph) noexcept
: __node_{__node}
, __graph_{__graph}
{
_CCCL_ASSERT(__node_ && __graph_, "construction of a graph_node_ref from a null cudaGraphNode_t handle");
}
//! \brief Compares two `graph_node_ref` objects for equality.
//!
//! \param __lhs The left-hand side `graph_node_ref` object to compare.
//! \param __rhs The right-hand side `graph_node_ref` object to compare.
//! \return `true` if both `graph_node_ref` objects are equal, `false` otherwise.
[[nodiscard]] _CCCL_HOST_API _CUDAX_CONSTEXPR_FRIEND bool
operator==(const graph_node_ref& __lhs, const graph_node_ref& __rhs) noexcept
{
return __lhs.__node_ == __rhs.__node_ && __lhs.__graph_ == __rhs.__graph_;
}
//! \brief Compares two `graph_node_ref` objects for inequality.
//!
//! \param __lhs The left-hand side `graph_node_ref` object to compare.
//! \param __rhs The right-hand side `graph_node_ref` object to compare.
//! \return `true` if both `graph_node_ref` objects are not equal, `false` otherwise.
[[nodiscard]] _CCCL_HOST_API _CUDAX_CONSTEXPR_FRIEND bool
operator!=(const graph_node_ref& __lhs, const graph_node_ref& __rhs) noexcept
{
return !(__lhs == __rhs);
}
//! \brief Test whether a `graph_node_ref` object is null.
//! \return `true` if `__rhs` is null, `false` otherwise.
[[nodiscard]] _CCCL_HOST_API _CUDAX_CONSTEXPR_FRIEND bool
operator==(::cuda::std::nullptr_t, const graph_node_ref& __rhs) noexcept
{
return !static_cast<bool>(__rhs);
}
//! \brief Test whether a `graph_node_ref` object is null.
//! \return `true` if `__rhs` is null, `false` otherwise.
[[nodiscard]] _CCCL_HOST_API _CUDAX_CONSTEXPR_FRIEND bool
operator==(const graph_node_ref& __lhs, ::cuda::std::nullptr_t) noexcept
{
return !static_cast<bool>(__lhs);
}
//! \brief Test whether a `graph_node_ref` object is not null.
//! \return `true` if `__rhs` is not null, `false` otherwise.
[[nodiscard]] _CCCL_HOST_API _CUDAX_CONSTEXPR_FRIEND bool
operator!=(::cuda::std::nullptr_t, const graph_node_ref& __rhs) noexcept
{
return static_cast<bool>(__rhs);
}
//! \brief Test whether a `graph_node_ref` object is not null.
//! \return `true` if `__lhs` is not null, `false` otherwise.
[[nodiscard]] _CCCL_HOST_API _CUDAX_CONSTEXPR_FRIEND bool
operator!=(const graph_node_ref& __lhs, ::cuda::std::nullptr_t) noexcept
{
return static_cast<bool>(__lhs);
}
//! \brief Checks if the graph node reference is valid.
//!
//! \details This operator allows the graph node reference to be used in a
//! boolean context to determine if it is valid. A valid graph node reference
//! is one where the internal node pointer is not null.
//!
//! \return `true` if the internal node pointer is not null, otherwise `false`.
[[nodiscard]] _CCCL_HOST_API explicit constexpr operator bool() const noexcept
{
return __node_ != nullptr;
}
//! \brief Checks if the graph node reference is not null.
//! \return `true` if the internal node pointer is null, otherwise `false`.
[[nodiscard]] _CCCL_HOST_API constexpr auto operator!() const noexcept -> bool
{
return !static_cast<bool>(*this);
}
//! \brief Swaps the contents of this graph_node_ref with another.
//! \param __other The other graph_node_ref to swap with.
_CCCL_HOST_API constexpr void swap(graph_node_ref& __other) noexcept
{
::cuda::std::swap(__node_, __other.__node_);
::cuda::std::swap(__graph_, __other.__graph_);
}
//! \brief Swaps the contents of two graph_node_ref objects.
//! \param __left The first graph_node_ref.
//! \param __right The second graph_node_ref.
_CCCL_HOST_API _CUDAX_CONSTEXPR_FRIEND void swap(graph_node_ref& __left, graph_node_ref& __right) noexcept
{
__left.swap(__right);
}
//! \brief Retrieves the underlying CUDA graph node.
//! \return The CUDA graph node.
[[nodiscard]] _CCCL_NODEBUG_HOST_API constexpr auto get() const noexcept -> cudaGraphNode_t
{
return __node_;
}
//! \brief Retrieves the CUDA graph this node belongs to.
//! \return The CUDA graph.
// internal for now because of a clash with get_graph() in path_builder. We could store the device in the
// graph_node_ref, but that feels like going a bit too far.
[[nodiscard]] _CCCL_NODEBUG_HOST_API constexpr auto get_native_graph_handle() const noexcept -> cudaGraph_t
{
return __graph_;
}
//! \brief Retrieves the type of the CUDA graph node.
//! \return The type of the graph node as a graph_node_type.
//! \pre The internal graph node handle is not null.
//! \throws If the CUDA API call to retrieve the node type fails.
[[nodiscard]] _CCCL_HOST_API auto type() const -> graph_node_type
{
_CCCL_ASSERT(__node_ != nullptr, "cannot get the type of a null graph node");
return static_cast<graph_node_type>(::cuda::experimental::__driver::__graphNodeGetType(__node_));
}
//! \brief Establishes dependencies between this node and other nodes.
//! This function sets up dependencies such that this node depends on the provided nodes.
//!
//! \tparam _Nodes Variadic template parameter for the types of the dependent nodes.
//! \param __nodes The nodes that this node depends on.
//! \pre The internal graph node handle is not null.
//! \throws If the CUDA API call to add dependencies fails.
template <class... _Nodes>
_CCCL_HOST_API constexpr void depends_on(const _Nodes&... __nodes)
{
cudaGraphNode_t __deps[]{__nodes.get()...};
return depends_on(::cuda::std::span{__deps});
}
//! \brief Establishes dependencies between this node and other nodes.
//! This function sets up dependencies such that this node depends on the provided nodes.
//!
//! \tparam _Node The type of the node to be added.
//! \tparam _Extent The extent of the span representing the dependencies.
//!
//! \param __deps A span of `cudaGraphNode_t` representing the dependencies of this node.
//! Each node in the span will become a dependency of this node.
//!
//! \throws cuda::std::cuda_error If the CUDA API call `cudaGraphAddDependencies` fails.
//!
//! \details
//! - This function first initializes an array of "dependant" nodes, where all dependant
//! nodes correspond to this node.
//! - The function then uses the CUDA API `cudaGraphAddDependencies` to establish the
//! dependencies between this node and the nodes provided in the `__deps` span.
//! - If the number of dependencies is small, a stack-allocated buffer is used; otherwise,
//! a dynamically allocated array is used to store the dependant nodes.
template <size_t _Extent>
_CCCL_HOST_API _CCCL_CONSTEXPR_CXX23 void depends_on(::cuda::std::span<cudaGraphNode_t, _Extent> __deps)
{
_CCCL_ASSERT(__node_ != nullptr, "cannot add dependencies to a null graph node");
if (!__deps.empty())
{
// Initialize an array of "dependant" nodes that correspond to the dependencies. All
// dependant nodes are __node_; thus, each node in __deps becomes a dependency of the
// newly created node.
using __src_arr_t = ::cuda::std::unique_ptr<cudaGraphNode_t[], void (*)(cudaGraphNode_t*) noexcept>;
cudaGraphNode_t __small_buffer[_Extent == ::cuda::std::dynamic_extent ? 4 : _Extent];
bool const __is_small = __deps.size() <= ::cuda::std::ranges::size(__small_buffer);
auto const __src_arr = __is_small ? __src_arr_t{__small_buffer, &__noop_deleter}
: __src_arr_t{::new cudaGraphNode_t[__deps.size()], &__array_deleter};
::cuda::std::fill(__src_arr.get(), __src_arr.get() + __deps.size(), __node_);
// Add the dependencies using __src_arr array and the span of dependencies.
::cuda::experimental::__driver::__graphAddDependencies(
__graph_,
__deps.data(), // dependencies
__src_arr.get(), // dependant nodes
__deps.size()); // number of dependencies
}
}
private:
friend struct graph_builder_ref;
template <class... _Nodes>
friend _CCCL_NODEBUG_HOST_API constexpr auto depends_on(const _Nodes&...) noexcept
-> ::cuda::std::array<cudaGraphNode_t, sizeof...(_Nodes)>;
_CCCL_NODEBUG_HOST_API explicit constexpr graph_node_ref(cudaGraphNode_t __node) noexcept
: __node_{__node}
{}
_CCCL_HOST_API static constexpr void __noop_deleter(cudaGraphNode_t*) noexcept {}
_CCCL_HOST_API static _CCCL_CONSTEXPR_CXX20_ALLOCATION void __array_deleter(cudaGraphNode_t* __ptr) noexcept
{
delete[] __ptr;
}
cudaGraphNode_t __node_ = nullptr; ///< The CUDA graph node.
cudaGraph_t __graph_ = nullptr; ///< The CUDA graph containing the node.
};
} // namespace cuda::experimental
#undef _CUDAX_CONSTEXPR_FRIEND
#include <cuda/std/__cccl/epilogue.h>
#endif // __CUDAX_GRAPH_GRAPH_NODE_REF

99
cccl_upstream/cudax/include/cuda/experimental/__graph/graph_node_type.cuh
View File

@@ -1,99 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDAX_GRAPH_GRAPH_NODE_TYPE
#define __CUDAX_GRAPH_GRAPH_NODE_TYPE
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda_runtime_api.h>
#include <cuda/std/__cccl/prologue.h>
namespace cuda::experimental
{
//! \enum graph_node_type
//!
//! \brief Represents the types of nodes that can exist in a CUDA graph.
//!
//! This enumeration defines various node types that can be used in CUDA graphs
//! to represent different operations or functionalities.
//!
//! \var graph_node_type::kernel
//! Represents a kernel execution node.
//!
//! \var graph_node_type::memcpy
//! Represents a memory copy operation node.
//!
//! \var graph_node_type::memset
//! Represents a memory set operation node.
//!
//! \var graph_node_type::host
//! Represents a host function execution node.
//!
//! \var graph_node_type::graph
//! Represents a nested graph node.
//!
//! \var graph_node_type::empty
//! Represents an empty node with no operation.
//!
//! \var graph_node_type::wait_event
//! Represents a node that waits for an event.
//!
//! \var graph_node_type::event_record
//! Represents a node that records an event.
//!
//! \var graph_node_type::semaphore_signal
//! Represents a node that signals an external semaphore.
//!
//! \var graph_node_type::semaphore_wait
//! Represents a node that waits on an external semaphore.
//!
//! \var graph_node_type::malloc
//! Represents a node that performs memory allocation.
//!
//! \var graph_node_type::free
//! Represents a node that performs memory deallocation.
//!
//! \var graph_node_type::conditional
//! Represents a conditional execution node.
enum class graph_node_type : int
{
kernel = cudaGraphNodeTypeKernel,
memcpy = cudaGraphNodeTypeMemcpy,
memset = cudaGraphNodeTypeMemset,
host = cudaGraphNodeTypeHost,
graph = cudaGraphNodeTypeGraph,
empty = cudaGraphNodeTypeEmpty,
wait_event = cudaGraphNodeTypeWaitEvent,
event_record = cudaGraphNodeTypeEventRecord,
semaphore_signal = cudaGraphNodeTypeExtSemaphoreSignal,
semaphore_wait = cudaGraphNodeTypeExtSemaphoreWait,
malloc = cudaGraphNodeTypeMemAlloc,
free = cudaGraphNodeTypeMemFree,
// batch_memop = CU_GRAPH_NODE_TYPE_BATCH_MEM_OP, // not exposed by the CUDA runtime
#if _CCCL_CTK_AT_LEAST(12, 8)
conditional = cudaGraphNodeTypeConditional
#endif // _CCCL_CTK_AT_LEAST(12, 8)
};
} // namespace cuda::experimental
#include <cuda/std/__cccl/epilogue.h>
#endif // __CUDAX_GRAPH_GRAPH_NODE_TYPE

142
cccl_upstream/cudax/include/cuda/experimental/__graph/host_launch.cuh
View File

@@ -1,142 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDAX__GRAPH_HOST_LAUNCH_CUH
#define _CUDAX__GRAPH_HOST_LAUNCH_CUH
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_CTK_AT_LEAST(12, 2)
# include <cuda/__launch/host_launch.h>
# include <cuda/std/__functional/invoke.h>
# include <cuda/std/__functional/reference_wrapper.h>
# include <cuda/std/__memory/addressof.h>
# include <cuda/std/__type_traits/is_function.h>
# include <cuda/std/__type_traits/is_move_constructible.h>
# include <cuda/std/__type_traits/is_pointer.h>
# include <cuda/std/__type_traits/remove_pointer.h>
# include <cuda/std/__utility/move.h>
# include <cuda/experimental/__driver/driver_api.cuh>
# include <cuda/experimental/__graph/graph_node_ref.cuh>
# include <cuda/experimental/__graph/path_builder.cuh>
# include <memory>
# include <cuda/std/__cccl/prologue.h>
namespace cuda::experimental
{
// Launcher for a plain no-arg function pointer.
template <class _FuncPtr>
_CCCL_HOST_API inline void CUDA_CB __graph_func_ptr_launcher(void* __callable_ptr)
{
reinterpret_cast<_FuncPtr>(__callable_ptr)();
}
// Launcher for a heap-allocated callable + argument pack. The graph host node callback
// signature is void(void*), unlike the stream callback which is void(CUstream, CUresult, void*),
// so we can't reuse ::cuda::__stream_callback_launcher here.
template <class _CallbackData>
_CCCL_HOST_API inline void CUDA_CB __graph_callback_launcher(void* __data_ptr)
{
auto* __data = static_cast<_CallbackData*>(__data_ptr);
// Copy, not move — the graph may be launched multiple times.
// Data lifetime is managed by a graph user object.
::cuda::std::apply(__data->__callable_, __data->__args_);
}
template <class _CallbackData>
_CCCL_HOST_API inline void CUDA_CB __graph_callback_data_destroyer(void* __data_ptr)
{
delete static_cast<_CallbackData*>(__data_ptr);
}
//! \brief Adds a host node to a CUDA graph path that invokes a callable on the host.
//!
//! The callable and its arguments are copied into a heap allocation whose lifetime is
//! tied to the graph via a CUDA user object. The graph can be launched multiple times.
//! The rules and restrictions match `cuda::host_launch`:
//! - The callable must not call into CUDA Runtime or Driver APIs.
//! - It must not depend on another thread that could block on asynchronous CUDA work.
//!
//! Three dispatch paths (mirroring `cuda::host_launch`):
//! 1. A bare no-arg function pointer; no allocation.
//! 2. A `std::reference_wrapper` (no args) passes the address of the referenced object.
//! 3. Everything else is heap-allocated with lifetime managed by a graph user object.
//!
//! \param __pb Path builder to insert the node into.
//! \param __callable Callable to execute on the host.
//! \param __args Arguments to forward to the callable.
//! \return A `graph_node_ref` for the newly added host node.
//! \throws cuda::std::cuda_error if node creation fails.
template <class _Callable, class... _Args>
_CCCL_HOST_API graph_node_ref host_launch(path_builder& __pb, _Callable __callable, _Args... __args)
{
static_assert(::cuda::std::is_invocable_v<_Callable, _Args...>,
"Callable can't be called with the supplied arguments");
static_assert(::cuda::std::is_move_constructible_v<_Callable>, "The callable must be move constructible");
static_assert((::cuda::std::is_move_constructible_v<_Args> && ...),
"All callback arguments must be move constructible");
constexpr bool __has_args = sizeof...(_Args) > 0;
::CUhostFn __fn = nullptr;
void* __user_data = nullptr;
if constexpr (!__has_args && ::cuda::std::is_pointer_v<_Callable>
&& ::cuda::std::is_function_v<::cuda::std::remove_pointer_t<_Callable>>)
{
__fn = __graph_func_ptr_launcher<_Callable>;
__user_data = reinterpret_cast<void*>(__callable);
}
else if constexpr (!__has_args && ::cuda::std::__is_cuda_std_reference_wrapper_v<_Callable>)
{
__fn = ::cuda::__host_func_launcher<typename _Callable::type>;
__user_data = static_cast<void*>(::cuda::std::addressof(__callable.get()));
}
else
{
// Heap-allocate the callback data. Lifetime is tied to the graph via a user object.
using _CallbackData = ::cuda::__stream_callback_data<_Callable, _Args...>;
auto __data = new _CallbackData{::cuda::std::move(__callable), {::cuda::std::move(__args)...}};
__fn = __graph_callback_launcher<_CallbackData>;
__user_data = __data;
::cuda::experimental::__driver::__graphRetainUserObject(
__pb.get_native_graph_handle(), __data, __graph_callback_data_destroyer<_CallbackData>);
}
auto __deps = __pb.get_dependencies();
::CUgraphNodeParams __params{};
__params.type = ::CU_GRAPH_NODE_TYPE_HOST;
__params.host.fn = __fn;
__params.host.userData = __user_data;
auto __node = ::cuda::experimental::__driver::__graphAddNode(
__pb.get_native_graph_handle(), __deps.data(), __deps.size(), &__params);
__pb.__clear_and_set_dependency_node(__node);
return graph_node_ref{__node, __pb.get_native_graph_handle()};
}
} // namespace cuda::experimental
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_CTK_AT_LEAST(12, 2)
#endif // _CUDAX__GRAPH_HOST_LAUNCH_CUH

244
cccl_upstream/cudax/include/cuda/experimental/__graph/path_builder.cuh
View File

@@ -1,244 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDAX__GRAPH_PATH_BUILDER_CUH
#define _CUDAX__GRAPH_PATH_BUILDER_CUH
#include <cuda/std/detail/__config>
#include <cuda/__event/event_ref.h>
#include <cuda/__runtime/api_wrapper.h>
#include <cuda/std/__exception/cuda_error.h>
#include <cuda/std/__exception/exception_macros.h>
#include <cuda/experimental/__driver/driver_api.cuh>
#include <cuda/experimental/__graph/concepts.cuh>
#include <cuda/experimental/__graph/graph_builder.cuh>
#include <cuda/experimental/__graph/graph_node_ref.cuh>
#include <cuda/experimental/__stream/stream_ref.cuh>
#include <vector>
#include <cuda_runtime.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
namespace cuda::experimental
{
//! \brief A builder for a path in a CUDA graph.
//!
//! This class allows for the creation of a path in a CUDA graph, which is a sequence of nodes that are executed in
//! order. The path builder can be used to add nodes to the path, and to set the dependencies between nodes. Thanks to
//! the sequential nature of the path builder, it is possible to write single code path that uses either a stream or a
//! path builder to result in either eager stream execution or construction of a lazy graph.
//!
//! \rst
//! .. _cudax-graph-path-builder:
//! \endrst
struct path_builder
{
//! \brief Construct a path builder that will insert nodes into a graph builder.
//! \param __builder The graph builder to create the path builder for.
_CCCL_HOST_API explicit path_builder(graph_builder_ref __builder)
: __dev_{__builder.get_device()}
, __graph_{__builder.get()}
{}
//! \brief Construct a path builder that will insert nodes into a graph.
//! \param __dev The device on which nodes inserted into the graph will execute.
//! \param __graph The graph to create the path builder for.
path_builder(device_ref __dev, cudaGraph_t __graph)
: __dev_{__dev}
, __graph_{__graph}
{}
#if _CCCL_CTK_AT_LEAST(12, 3)
//! \brief Capture the nodes into the path builder from a legacy stream capture.
//! \param __stream The stream to use for the capture.
//! \param __capture_fn A function that will be called with the stream to capture the nodes to.
template <typename _Fn>
_CCCL_HOST_API void legacy_stream_capture(stream_ref __stream, _Fn&& __capture_fn)
{
::cuda::experimental::__driver::__streamBeginCaptureToGraph(
__stream.get(), __graph_, __nodes_.data(), __nodes_.size(), CU_STREAM_CAPTURE_MODE_GLOBAL);
__capture_fn(__stream.get());
auto __info = ::cuda::experimental::__driver::__streamGetCaptureInfo(__stream.get());
if (__info.__status != CU_STREAM_CAPTURE_STATUS_ACTIVE)
{
_CCCL_THROW(cuda::cuda_error, cudaErrorInvalidValue, "Stream capture no longer active", "cuStreamGetCaptureInfo");
}
cudaGraph_t __graph_out = nullptr;
::cuda::experimental::__driver::__streamEndCapture(__stream.get(), &__graph_out);
assert(__graph_out == __graph_);
assert(__info.__ndeps == 1);
__nodes_.clear();
__nodes_.push_back(__info.__deps[0]);
}
#endif // _CCCL_CTK_AT_LEAST(12, 3)
//! \brief Clear the path builder and set the dependency node.
//! Used by most APIs that operate on a path builder to insert a new node into the path.
//! \param __node The node to set as the dependency node.
_CCCL_HOST_API void __clear_and_set_dependency_node(cudaGraphNode_t __node)
{
__nodes_.clear(); // Clear existing nodes
__nodes_.push_back(__node);
}
//! \brief Get the dependencies of the path builder.
//! \return A span of the dependencies of the path builder.
[[nodiscard]] _CCCL_NODEBUG_HOST_API auto get_dependencies() const noexcept
-> ::cuda::std::span<const cudaGraphNode_t>
{
return ::cuda::std::span(__nodes_.data(), __nodes_.size());
}
//! \brief Add the dependencies of another path builder to this path builder.
//! \param __other The path builder to add dependencies from.
//! Named wait to match the stream/stream_ref wait function
_CCCL_HOST_API void wait(const path_builder& __other)
{
__nodes_.insert(__nodes_.end(), __other.__nodes_.begin(), __other.__nodes_.end());
}
//! \brief Adds an event-wait node and makes it the next dependency.
//!
//! The new node waits for \p __ev to be recorded before any successor of this node
//! executes. This mirrors `stream_ref::wait(event_ref)`.
//!
//! \param __ev Event that graph execution should wait on.
//! \return A `graph_node_ref` for the newly added event-wait node.
//! \throws cuda::std::cuda_error if `cudaGraphAddEventWaitNode` fails.
#if _CCCL_CTK_AT_LEAST(12, 2)
_CCCL_HOST_API graph_node_ref wait(::cuda::event_ref __ev)
{
auto __deps = get_dependencies();
::CUgraphNodeParams __params{};
__params.type = ::CU_GRAPH_NODE_TYPE_WAIT_EVENT;
__params.eventWait.event = __ev.get();
auto __node = ::cuda::experimental::__driver::__graphAddNode(__graph_, __deps.data(), __deps.size(), &__params);
__clear_and_set_dependency_node(__node);
return graph_node_ref{__node, __graph_};
}
//! \brief Adds an event-record node and makes it the next dependency.
//!
//! The new node records \p __ev when it executes, signalling any downstream waiters.
//! This mirrors `stream_ref::record_event(event_flags)` but takes an existing event
//! because graph construction happens before execution.
//!
//! \param __ev Event to record.
//! \return A `graph_node_ref` for the newly added event-record node.
//! \throws cuda::std::cuda_error if `cudaGraphAddEventRecordNode` fails.
_CCCL_HOST_API graph_node_ref record_event(::cuda::event_ref __ev)
{
auto __deps = get_dependencies();
::CUgraphNodeParams __params{};
__params.type = ::CU_GRAPH_NODE_TYPE_EVENT_RECORD;
__params.eventRecord.event = __ev.get();
auto __node = ::cuda::experimental::__driver::__graphAddNode(__graph_, __deps.data(), __deps.size(), &__params);
__clear_and_set_dependency_node(__node);
return graph_node_ref{__node, __graph_};
}
#endif // _CCCL_CTK_AT_LEAST(12, 2)
template <typename... Nodes>
static constexpr bool __all_dependencies = (graph_dependency<Nodes> && ...);
//! \brief Add the dependencies of another path builder or single nodes to this path builder.
//! \param __nodes The nodes or path builders to add to the path builder as dependencies.
_CCCL_TEMPLATE(typename... Nodes)
_CCCL_REQUIRES(__all_dependencies<Nodes...>)
_CCCL_HOST_API void depends_on(Nodes&&... __nodes)
{
(
[this](auto&& __arg) {
if constexpr (::cuda::std::is_same_v<::cuda::std::decay_t<decltype(__arg)>, graph_node_ref>)
{
__nodes_.push_back(__arg.get());
}
else
{
__nodes_.insert(__nodes_.end(), __arg.__nodes_.begin(), __arg.__nodes_.end());
}
}(static_cast<Nodes&&>(__nodes)),
...);
}
//! \brief Get the graph that the path builder is building.
//! \return The graph that the path builder is building.
[[nodiscard]] _CCCL_NODEBUG_HOST_API constexpr auto get_graph() const noexcept -> graph_builder_ref
{
return graph_builder_ref(__graph_, __dev_);
}
//! \internal
//! Internal graph handle getter to match graph_node_ref::__get_graph().
[[nodiscard]] _CCCL_NODEBUG_HOST_API constexpr auto get_native_graph_handle() const noexcept -> cudaGraph_t
{
return __graph_;
}
//! \brief Retrieves the device on which graph nodes inserted by the path builder will execute.
//! \return The device on which graph nodes inserted by the path builder will execute.
[[nodiscard]] _CCCL_NODEBUG_HOST_API constexpr auto get_device() const noexcept -> device_ref
{
return __dev_;
}
private:
device_ref __dev_;
cudaGraph_t __graph_;
// TODO should this be a custom class that does inline storage for small counts?
::std::vector<cudaGraphNode_t> __nodes_;
};
//! \brief Create a new path builder for a graph builder.
//! \param __gb The graph builder to create the path builder for.
//! \param __nodes The nodes the path builder will depend on.
//! \return A new path builder for the graph builder.
template <typename... Nodes>
[[nodiscard]] _CCCL_HOST_API path_builder start_path(graph_builder_ref __gb, Nodes... __nodes)
{
path_builder __pb(__gb);
if constexpr (sizeof...(__nodes) > 0)
{
__pb.depends_on(__nodes...);
}
return __pb;
}
//! \brief Create a new path builder for a device and a first node.
//! \param __dev The device to create the path builder for.
//! \param __first_node At least one node that the path builder will depend on.
//! \param __nodes Additional nodes that the path builder will depend on.
//! \return A new path builder for the device and the first node.
template <typename _FirstNode, typename... _Nodes>
[[nodiscard]] _CCCL_HOST_API path_builder start_path(device_ref __dev, _FirstNode __first_node, _Nodes... __nodes)
{
path_builder __pb(__dev, __first_node.get_native_graph_handle());
__pb.depends_on(__first_node, __nodes...);
return __pb;
}
} // namespace cuda::experimental
#endif // _CUDAX__GRAPH_PATH_BUILDER_CUH