[CCCL] 瘦身 + 补全: 移除 cudax/python/libcudacxx-tests 冗余文件, 新增 c2h 测试助手 + cmake 构建系统 + 8 个 CUDA thrust examples
变更摘要:
- 删除: cudax/ (783 files, 7.2M) — 实验性组件，竞赛不需要
- 删除: python/ (226 files, 2.0M) — Python 绑定，竞赛不需要
- 删除: libcudacxx/{test,benchmarks,codegen,cmake,share} (4432 files, 31M)
保留: libcudacxx/include/ (1463 headers, cuda::std 编译依赖)
- 新增: c2h/ (27 files) — CUB Catch2 测试辅助头文件，编译 243 个测试必需
- 新增: cmake/ (29 files) — CCCL 原生 CMake 构建系统
- 新增: thrust/examples/cuda/ (7 files) + cpp_integration/ (1 file)
async_reduce, custom_temporary_allocation, explicit_cuda_stream,
global_device_vector, range_view, unwrap_pointer, wrap_pointer, device
结果: cccl_upstream 从 74M→35M (瘦身 53%), 核心内容 100% 保留:
27/27 tuning headers, 78 benchmarks, 243 tests,
60 thrust examples, 18 CUB examples, 全部编译头文件
This commit is contained in:
muh-bot
@@ -1,145 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_ANY_ALLOCATOR
|
||||
#define __CUDAX_EXECUTION_ANY_ALLOCATOR
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__type_traits/is_specialization_of.h>
|
||||
#include <cuda/__utility/basic_any.h>
|
||||
#include <cuda/std/__fwd/optional.h>
|
||||
#include <cuda/std/__memory/allocator.h>
|
||||
#include <cuda/std/__memory/allocator_traits.h>
|
||||
#include <cuda/std/__type_traits/decay.h>
|
||||
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
template <class _Value>
|
||||
struct any_allocator;
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
template <class _Allocator, class _Value = typename _Allocator::value_type>
|
||||
_CCCL_PUBLIC_API auto __any_allocator_allocate(_Allocator& __alloc, size_t __count) -> _Value*
|
||||
{
|
||||
return ::cuda::std::allocator_traits<_Allocator>::allocate(__alloc, __count);
|
||||
}
|
||||
|
||||
template <class _Allocator, class _Value = typename _Allocator::value_type>
|
||||
_CCCL_PUBLIC_API void __any_allocator_deallocate(_Allocator& __alloc, _Value* __ptr, size_t __count) noexcept
|
||||
{
|
||||
::cuda::std::allocator_traits<_Allocator>::deallocate(__alloc, static_cast<_Value*>(__ptr), __count);
|
||||
}
|
||||
|
||||
template <class...>
|
||||
struct __iallocator : __basic_interface<__iallocator, ::cuda::__extends<::cuda::__icopyable<>>>
|
||||
{
|
||||
using value_type = ::cuda::std::byte;
|
||||
|
||||
template <class _Other>
|
||||
struct rebind
|
||||
{
|
||||
static_assert(__same_as<_Other, value_type>);
|
||||
using other = __iallocator;
|
||||
};
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto allocate(size_t __bytes) -> value_type*
|
||||
{
|
||||
constexpr auto __allocate_vfn = &__any_allocator_allocate<__iallocator<>>;
|
||||
return ::cuda::__virtcall<__allocate_vfn>(this, __bytes);
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void deallocate(value_type* __ptr, size_t __bytes) noexcept
|
||||
{
|
||||
constexpr auto __deallocate_vfn = &__any_allocator_deallocate<__iallocator<>>;
|
||||
::cuda::__virtcall<__deallocate_vfn>(this, __ptr, __bytes);
|
||||
}
|
||||
|
||||
template <class _Allocator>
|
||||
using overrides =
|
||||
__overrides_for<_Allocator, &__any_allocator_allocate<_Allocator>, &__any_allocator_deallocate<_Allocator>>;
|
||||
};
|
||||
|
||||
using __any_allocator = ::cuda::__basic_any<__iallocator<>>;
|
||||
|
||||
template <class _Allocator>
|
||||
_CCCL_CONCEPT __is_any_allocator = __is_specialization_of_v<_Allocator, execution::any_allocator>;
|
||||
} // namespace __detail
|
||||
|
||||
template <class _Value>
|
||||
struct any_allocator : private __detail::__any_allocator
|
||||
{
|
||||
using value_type = _Value;
|
||||
|
||||
template <class _Other>
|
||||
struct rebind
|
||||
{
|
||||
using other = any_allocator<_Other>;
|
||||
};
|
||||
|
||||
_CCCL_HOST_DEVICE_API any_allocator(::cuda::std::allocator<void>) noexcept
|
||||
: __detail::__any_allocator{::cuda::std::allocator<::cuda::std::byte>{}}
|
||||
{}
|
||||
|
||||
_CCCL_TEMPLATE(class _Allocator)
|
||||
_CCCL_REQUIRES((!__detail::__is_any_allocator<_Allocator>) //
|
||||
_CCCL_AND(!::cuda::std::__is_cuda_std_optional_v<_Allocator>)
|
||||
_CCCL_AND ::cuda::__satisfies<_Allocator, __detail::__iallocator<>>)
|
||||
_CCCL_HOST_DEVICE_API any_allocator(_Allocator __alloc)
|
||||
: __detail::__any_allocator{__byte_allocator_t<_Allocator>(static_cast<_Allocator&&>(__alloc))}
|
||||
{}
|
||||
|
||||
_CCCL_TEMPLATE(class _OtherValue)
|
||||
_CCCL_REQUIRES(__not_same_as<_OtherValue, _Value>)
|
||||
_CCCL_HOST_DEVICE_API any_allocator(any_allocator<_OtherValue> __other) noexcept
|
||||
: __detail::__any_allocator{static_cast<__detail::__any_allocator&&>(__other)}
|
||||
{}
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto allocate(size_t __count) -> _Value*
|
||||
{
|
||||
return reinterpret_cast<_Value*>(this->__basic_any::allocate(__count * sizeof(_Value)));
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void deallocate(_Value* __ptr, size_t __count) noexcept
|
||||
{
|
||||
this->__basic_any::deallocate(reinterpret_cast<::cuda::std::byte*>(__ptr), __count * sizeof(_Value));
|
||||
}
|
||||
|
||||
private:
|
||||
template <class>
|
||||
friend struct any_allocator;
|
||||
|
||||
template <class _Allocator>
|
||||
using __byte_allocator_t = ::cuda::std::__rebind_alloc<::cuda::std::allocator_traits<_Allocator>, ::cuda::std::byte>;
|
||||
};
|
||||
|
||||
template <class _Allocator>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES any_allocator(_Allocator) -> any_allocator<typename _Allocator::value_type>;
|
||||
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES any_allocator(::cuda::std::allocator<void>) -> any_allocator<::cuda::std::byte>;
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_ANY_ALLOCATOR
|
||||
@@ -1,84 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_APPLY_SENDER
|
||||
#define __CUDAX_EXECUTION_APPLY_SENDER
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__type_traits/conditional.h>
|
||||
#include <cuda/std/__type_traits/is_valid_expansion.h>
|
||||
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/domain.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
//! A callable object that implements the `std::execution::apply_sender` functionality.
|
||||
//! This is used to apply a sender to a domain, tag, and arguments, as specified in the
|
||||
//! C++ standard draft. The implementation ensures compatibility with CUDA C++ Core
|
||||
//! Libraries.
|
||||
//! @see https://eel.is/c++draft/exec.snd.apply
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT apply_sender_t
|
||||
{
|
||||
private:
|
||||
//! A type alias that determines the domain to apply the sender to. If the expansion of
|
||||
//! `__apply_sender_result_t` is valid for the given domain and arguments, the domain is
|
||||
//! used; otherwise, the `default_domain` is used.
|
||||
//! @tparam _Domain The domain to check.
|
||||
//! @tparam _Args The arguments to validate against the domain.
|
||||
template <class _Domain, class... _Args>
|
||||
using __apply_domain_t _CCCL_NODEBUG_ALIAS = ::cuda::std::
|
||||
_If<::cuda::std::_IsValidExpansion<__apply_sender_result_t, _Domain, _Args...>::value, _Domain, default_domain>;
|
||||
|
||||
public:
|
||||
//! Applies a sender to a domain, tag, and arguments.
|
||||
//! @tparam _Domain The domain used to select the algorithm implementation.
|
||||
//! @tparam _Tag The tag associated with the algorithm.
|
||||
//! @tparam _Sndr The sender to be applied.
|
||||
//! @tparam _Args The arguments to pass to the algorithm.
|
||||
//! @param __sndr The sender object.
|
||||
//! @param __args The arguments to pass to the algorithm.
|
||||
//! @return `DOM{}.apply_sender(_Tag{}, __sndr, __args...)`, where `DOM` is the first of
|
||||
//! [`_Domain`, `default_domain`] to make the expression well-formed.
|
||||
//! @note This function is `constexpr` and `noexcept` if the underlying domain's
|
||||
//! `apply_sender` is `noexcept`.
|
||||
//! @throws Any exception thrown by the underlying domain's `apply_sender`.
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Domain, class _Tag, class _Sndr, class... _Args>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Domain, _Tag, _Sndr&& __sndr, _Args&&... __args) const
|
||||
noexcept(noexcept(__apply_domain_t<_Domain, _Tag, _Sndr, _Args...>{}.apply_sender(
|
||||
_Tag{}, static_cast<_Sndr&&>(__sndr), static_cast<_Args&&>(__args)...)))
|
||||
-> __apply_sender_result_t<__apply_domain_t<_Domain, _Tag, _Sndr, _Args...>, _Tag, _Sndr, _Args...>
|
||||
{
|
||||
using __dom_t _CCCL_NODEBUG_ALIAS = __apply_domain_t<_Domain, _Tag, _Sndr, _Args...>;
|
||||
//! Calls the algorithm specified by _Tag using the determined domain.
|
||||
return __dom_t{}.apply_sender(_Tag{}, static_cast<_Sndr&&>(__sndr), static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
};
|
||||
|
||||
//! A global constant instance of `apply_sender_t`.
|
||||
//! This can be used directly to invoke the `apply_sender` functionality.
|
||||
_CCCL_GLOBAL_CONSTANT apply_sender_t apply_sender{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_APPLY_SENDER
|
||||
@@ -1,83 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
// Copyright (c) 2023 Maikel Nadolski
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_ATOMIC_INTRUSIVE_QUEUE
|
||||
#define __CUDAX_EXECUTION_ATOMIC_INTRUSIVE_QUEUE
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/atomic>
|
||||
|
||||
#include <cuda/experimental/__execution/intrusive_queue.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
// An atomic queue that supports multiple producers and a single consumer.
|
||||
template <auto _NextPtr>
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT __atomic_intrusive_queue;
|
||||
|
||||
template <class _Tp, _Tp* _Tp::* _NextPtr>
|
||||
class alignas(64) __atomic_intrusive_queue<_NextPtr>
|
||||
{
|
||||
public:
|
||||
_CCCL_HOST_DEVICE_API auto push(_Tp* __node) noexcept -> bool
|
||||
{
|
||||
_CCCL_ASSERT(__node != nullptr, "Cannot push a null pointer to the queue");
|
||||
_Tp* __old_head = __head_.load(::cuda::std::memory_order_relaxed);
|
||||
do
|
||||
{
|
||||
__node->*_NextPtr = __old_head;
|
||||
} while (!__head_.compare_exchange_weak(__old_head, __node, ::cuda::std::memory_order_acq_rel));
|
||||
|
||||
// If the queue was empty before, we notify the consumer thread that there is now an
|
||||
// item available. If the queue was not empty, we do not notify, because the consumer
|
||||
// thread has already been notified.
|
||||
if (__old_head != nullptr)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// There can be only one consumer thread, so we can use notify_one here instead of
|
||||
// notify_all:
|
||||
__head_.notify_one();
|
||||
return true;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void wait_for_item() noexcept
|
||||
{
|
||||
// Wait until the queue has an item in it:
|
||||
__head_.wait(nullptr);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API auto pop_all() noexcept -> __intrusive_queue<_NextPtr>
|
||||
{
|
||||
auto* const __list = __head_.exchange(nullptr, ::cuda::std::memory_order_acquire);
|
||||
return __intrusive_queue<_NextPtr>::make_reversed(__list);
|
||||
}
|
||||
|
||||
private:
|
||||
::cuda::std::atomic<_Tp*> __head_{nullptr};
|
||||
};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_ATOMIC_INTRUSIVE_QUEUE
|
||||
@@ -1,472 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_BULK
|
||||
#define __CUDAX_EXECUTION_BULK
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__cmath/ceil_div.h>
|
||||
#include <cuda/__launch/configuration.h>
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__concepts/arithmetic.h>
|
||||
#include <cuda/std/__concepts/same_as.h>
|
||||
#include <cuda/std/__exception/exception_macros.h>
|
||||
#include <cuda/std/__tuple_dir/ignore.h>
|
||||
#include <cuda/std/__type_traits/is_callable.h>
|
||||
#include <cuda/std/__type_traits/is_same.h>
|
||||
#include <cuda/std/__type_traits/is_void.h>
|
||||
#include <cuda/std/__utility/forward_like.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/concepts.cuh>
|
||||
#include <cuda/experimental/__execution/domain.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/exception.cuh>
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/get_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/policy.cuh>
|
||||
#include <cuda/experimental/__execution/queries.cuh>
|
||||
#include <cuda/experimental/__execution/rcvr_ref.cuh>
|
||||
#include <cuda/experimental/__execution/transform_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/transform_sender.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(4702) // warning: unreachable code
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
namespace __bulk
|
||||
{
|
||||
template <class _Shape, class _Fn, class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __state_t
|
||||
{
|
||||
_Rcvr __rcvr_;
|
||||
_Shape __shape_;
|
||||
_Fn __fn_;
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// attributes for bulk senders
|
||||
template <class _Sndr, class _Shape>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __attrs_t
|
||||
{
|
||||
[[nodiscard]] _CCCL_HOST_API static constexpr auto __get_launch_config(_Shape __shape) noexcept
|
||||
{
|
||||
constexpr int __threads_per_block = 256;
|
||||
const int __grid_blocks = ::cuda::ceil_div(static_cast<int>(__shape), __threads_per_block);
|
||||
auto __dims = ::cuda::make_hierarchy(block_dims<__threads_per_block>(), grid_dims(__grid_blocks));
|
||||
return make_config(__dims, cooperative_launch());
|
||||
}
|
||||
|
||||
using __launch_config_t = decltype(__get_launch_config(_Shape()));
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_launch_config_t) const noexcept -> __launch_config_t
|
||||
{
|
||||
NV_IF_ELSE_TARGET(NV_IS_HOST, (return __get_launch_config(__shape_);), ({
|
||||
_CCCL_ASSERT(false, "cannot get a launch configuration from device");
|
||||
::cuda::std::terminate();
|
||||
}))
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Query, class... _Args)
|
||||
_CCCL_REQUIRES(__forwarding_query<_Query> _CCCL_AND __queryable_with<env_of_t<_Sndr>, _Query, _Args...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(_Query, _Args&&... __args) const
|
||||
noexcept(__nothrow_queryable_with<env_of_t<_Sndr>, _Query, _Args...>)
|
||||
-> __query_result_t<env_of_t<_Sndr>, _Query, _Args...>
|
||||
{
|
||||
return execution::get_env(__sndr_).query(_Query{}, static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
_Shape __shape_;
|
||||
const _Sndr& __sndr_;
|
||||
};
|
||||
} // namespace __bulk
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// generic bulk utilities
|
||||
template <class _BulkTag>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __bulk_t
|
||||
{
|
||||
// This is a function object that is used to transform the value completion signatures
|
||||
// of a bulk sender's child operation. It does type checking and "throws" if the bulk
|
||||
// function is not callable with the value datums of the predecessor.
|
||||
template <class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __transform_value_completion_fn
|
||||
{
|
||||
template <class... _Ts>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto operator()() const
|
||||
{
|
||||
// The function objects passed to the "chunked" and "unchunked" flavors of bulk have
|
||||
// different signatures, so we need to type-check them separately.
|
||||
if constexpr (_BulkTag::__is_chunked())
|
||||
{
|
||||
if constexpr (__callable<_Fn&, _Shape, _Shape, _Ts&...>)
|
||||
{
|
||||
return completion_signatures<set_value_t(_Ts...)>{}
|
||||
+ __eptr_completion_if<!__nothrow_callable<_Fn&, _Shape, _Shape, _Ts&...>>();
|
||||
}
|
||||
else
|
||||
{
|
||||
return invalid_completion_signature<_WHERE(_IN_ALGORITHM, _BulkTag),
|
||||
_WHAT(_FUNCTION_IS_NOT_CALLABLE),
|
||||
_WITH_FUNCTION(_Fn&),
|
||||
_WITH_ARGUMENTS(_Shape, _Shape, _Ts & ...)>();
|
||||
}
|
||||
}
|
||||
else if constexpr (__callable<_Fn&, _Shape, _Ts&...>)
|
||||
{
|
||||
return completion_signatures<set_value_t(_Ts...)>{}
|
||||
+ __eptr_completion_if<!__nothrow_callable<_Fn&, _Shape, _Ts&...>>();
|
||||
}
|
||||
else
|
||||
{
|
||||
return invalid_completion_signature<_WHERE(_IN_ALGORITHM, _BulkTag),
|
||||
_WHAT(_FUNCTION_IS_NOT_CALLABLE),
|
||||
_WITH_FUNCTION(_Fn&),
|
||||
_WITH_ARGUMENTS(_Shape, _Ts & ...)>();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Shape, class _Fn, class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_base_t
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_error(_Error&& __err) noexcept
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(__state_->__rcvr_), static_cast<_Error&&>(__err));
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_stopped() noexcept
|
||||
{
|
||||
execution::set_stopped(static_cast<_Rcvr&&>(__state_->__rcvr_));
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __fwd_env_t<env_of_t<_Rcvr>>
|
||||
{
|
||||
return __fwd_env(execution::get_env(__state_->__rcvr_));
|
||||
}
|
||||
|
||||
__bulk::__state_t<_Shape, _Fn, _Rcvr>* __state_;
|
||||
};
|
||||
|
||||
// This is the operation state for bulk senders. It connects the child sender with
|
||||
// a receiver defined by _BulkTag.
|
||||
template <class _CvSndr, class _Shape, class _Fn, class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
using __rcvr_t = typename _BulkTag::template __rcvr_t<_Shape, _Fn, _Rcvr>;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __opstate_t(_CvSndr&& __sndr, _Rcvr __rcvr, _Shape __shape, _Fn __fn)
|
||||
: __state_{static_cast<_Rcvr&&>(__rcvr), __shape, static_cast<_Fn&&>(__fn)}
|
||||
, __opstate_{execution::connect(static_cast<_CvSndr&&>(__sndr), __rcvr_t{{&__state_}})}
|
||||
{}
|
||||
|
||||
_CCCL_IMMOVABLE(__opstate_t);
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void start() noexcept
|
||||
{
|
||||
execution::start(__opstate_);
|
||||
}
|
||||
|
||||
__bulk::__state_t<_Shape, _Fn, _Rcvr> __state_;
|
||||
connect_result_t<_CvSndr, __rcvr_t> __opstate_;
|
||||
};
|
||||
|
||||
template <class _Policy, class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_base_t
|
||||
{
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr&& __sndr) &&
|
||||
{
|
||||
static_assert(__is_sender<_Sndr>);
|
||||
|
||||
if constexpr (!dependent_sender<_Sndr>)
|
||||
{
|
||||
using __sndr_t = typename _BulkTag::template __sndr_t<_Sndr, _Policy, _Shape, _Fn>;
|
||||
__assert_valid_completion_signatures(execution::get_completion_signatures<__sndr_t>());
|
||||
}
|
||||
|
||||
return typename _BulkTag::template __sndr_t<_Sndr, _Policy, _Shape, _Fn>{
|
||||
{{}, static_cast<__closure_base_t&&>(*this), static_cast<_Sndr&&>(__sndr)}};
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr&& __sndr) const&
|
||||
{
|
||||
return __closure_base_t(*this)(static_cast<_Sndr&&>(__sndr));
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr auto operator|(_Sndr&& __sndr, __closure_base_t __self)
|
||||
{
|
||||
return static_cast<__closure_base_t&&>(__self)(static_cast<_Sndr&&>(__sndr));
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ _Policy __policy_;
|
||||
_Shape __shape_;
|
||||
_Fn __fn_;
|
||||
};
|
||||
|
||||
// This is the sender type for the three bulk algorithms.
|
||||
template <class _Sndr, class _Policy, class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_base_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class _Self, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
_CUDAX_LET_COMPLETIONS(
|
||||
auto(__child_completions) = execution::get_child_completion_signatures<_Self, _Sndr, _Env...>())
|
||||
{
|
||||
return transform_completion_signatures(__child_completions, __transform_value_completion_fn<_Shape, _Fn>{});
|
||||
}
|
||||
}
|
||||
|
||||
// The bulk algorithm lowers to a bulk_chunked sender. The bulk sender itself should
|
||||
// not have `connect` functions, since they should never be called. Hence, we
|
||||
// constrain these functions with !same_as<_BulkTag, bulk_t>.
|
||||
_CCCL_TEMPLATE(class _Rcvr)
|
||||
_CCCL_REQUIRES((!::cuda::std::same_as<_BulkTag, bulk_t>) )
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) && -> __opstate_t<_Sndr, _Shape, _Fn, _Rcvr>
|
||||
{
|
||||
return __opstate_t<_Sndr, _Shape, _Fn, _Rcvr>{
|
||||
static_cast<_Sndr&&>(__sndr_),
|
||||
static_cast<_Rcvr&&>(__rcvr),
|
||||
__state_.__shape_,
|
||||
static_cast<_Fn&&>(__state_.__fn_)};
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Rcvr)
|
||||
_CCCL_REQUIRES((!::cuda::std::same_as<_BulkTag, bulk_t>) )
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
connect(_Rcvr __rcvr) const& -> __opstate_t<const _Sndr&, _Shape, _Fn, _Rcvr>
|
||||
{
|
||||
return __opstate_t<const _Sndr&, _Shape, _Fn, _Rcvr>{
|
||||
__sndr_, static_cast<_Rcvr&&>(__rcvr), __state_.__shape_, __state_.__fn_};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __bulk::__attrs_t<_Sndr, _Shape>
|
||||
{
|
||||
return {__state_.__shape_, __sndr_};
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ _BulkTag __tag_;
|
||||
__closure_base_t<_Policy, _Shape, _Fn> __state_;
|
||||
_Sndr __sndr_;
|
||||
};
|
||||
|
||||
// This function call operator is the entry point for the bulk algorithms. It takes a
|
||||
// predecessor sender, a policy, a shape, and a function, and returns a sender that can
|
||||
// be connected to a receiver.
|
||||
template <class _Sndr, class _Policy, class _Shape, class _Fn>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
operator()(_Sndr&& __sndr, _Policy __policy, _Shape __shape, _Fn __fn) const
|
||||
{
|
||||
return (static_cast<_Sndr&&>(__sndr) | (*this)(__policy, __shape, static_cast<_Fn&&>(__fn)));
|
||||
}
|
||||
|
||||
// This function call operator creates a sender adaptor closure object that can appear
|
||||
// on the right-hand side of a pipe operator, like: sndr | bulk(par, shape, fn).
|
||||
template <class _Policy, class _Shape, class _Fn>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto operator()(_Policy __policy, _Shape __shape, _Fn __fn) const
|
||||
{
|
||||
static_assert(::cuda::std::integral<_Shape>);
|
||||
static_assert(::cuda::std::is_execution_policy_v<_Policy>);
|
||||
using __closure_t = typename _BulkTag::template __closure_t<_Policy, _Shape, _Fn>;
|
||||
return __closure_t{{__policy, __shape, static_cast<_Fn&&>(__fn)}};
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// bulk_chunked
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT bulk_chunked_t : __bulk_t<bulk_chunked_t>
|
||||
{
|
||||
template <class _Sndr, class _Policy, class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t : __bulk_t::__sndr_base_t<_Sndr, _Policy, _Shape, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Policy, class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t : __bulk_t::__closure_base_t<_Policy, _Shape, _Fn>
|
||||
{};
|
||||
|
||||
// This is the receiver for the bulk_chunked sender. It provides the implementation for
|
||||
// `set_value` that calls the function with the begin and end shapes, and the value
|
||||
// results of the predecessor.
|
||||
template <class _Shape, class _Fn, class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_t : __bulk_t::__rcvr_base_t<_Shape, _Fn, _Rcvr>
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class... _Values>
|
||||
_CCCL_HOST_DEVICE_API void set_value(_Values&&... __values) noexcept
|
||||
{
|
||||
_CCCL_TRY //
|
||||
{
|
||||
this->__state_->__fn_(_Shape(0), _Shape(this->__state_->__shape_), __values...);
|
||||
execution::set_value(static_cast<_Rcvr&&>(this->__state_->__rcvr_), static_cast<_Values&&>(__values)...);
|
||||
}
|
||||
_CCCL_CATCH_ALL //
|
||||
{
|
||||
if constexpr (!__nothrow_callable<_Fn&, _Shape, _Shape, _Values&...>)
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(this->__state_->__rcvr_), execution::current_exception());
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr bool __is_chunked() noexcept
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto bulk_chunked = bulk_chunked_t{};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// bulk_unchunked
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT bulk_unchunked_t : __bulk_t<bulk_unchunked_t>
|
||||
{
|
||||
// This is the receiver for the bulk_unchunked sender. It provides the implementation
|
||||
// for `set_value` that calls the function repeatedly with an index and the value
|
||||
// results of the predecessor. The index is monotonically increasing from 0 to the shape
|
||||
// minus one.
|
||||
template <class _Shape, class _Fn, class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_t : __bulk_t::__rcvr_base_t<_Shape, _Fn, _Rcvr>
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class... _Values>
|
||||
_CCCL_HOST_DEVICE_API void set_value(_Values&&... __values) noexcept
|
||||
{
|
||||
_CCCL_TRY //
|
||||
{
|
||||
for (_Shape __index{}; __index != this->__state_->__shape_; ++__index)
|
||||
{
|
||||
this->__state_->__fn_(_Shape(__index), __values...);
|
||||
}
|
||||
execution::set_value(static_cast<_Rcvr&&>(this->__state_->__rcvr_), static_cast<_Values&&>(__values)...);
|
||||
}
|
||||
_CCCL_CATCH_ALL //
|
||||
{
|
||||
if constexpr (!__nothrow_callable<_Fn&, _Shape, _Values&...>)
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(this->__state_->__rcvr_), execution::current_exception());
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Sndr, class _Policy, class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t : __bulk_t::__sndr_base_t<_Sndr, _Policy, _Shape, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Policy, class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t : __bulk_t::__closure_base_t<_Policy, _Shape, _Fn>
|
||||
{};
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr bool __is_chunked() noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto bulk_unchunked = bulk_unchunked_t{};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// bulk
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT bulk_t : __bulk_t<bulk_t>
|
||||
{
|
||||
template <class _Sndr, class _Policy, class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t : __bulk_t::__sndr_base_t<_Sndr, _Policy, _Shape, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Policy, class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t : __bulk_t::__closure_base_t<_Policy, _Shape, _Fn>
|
||||
{};
|
||||
|
||||
// This is a function adaptor that transforms a `bulk` function that takes a single
|
||||
// shape to a `bulk_chunked` function that takes a begin and end shape.
|
||||
template <class _Shape, class _Fn>
|
||||
struct __bulk_chunked_fn
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API void operator()(_Shape __begin, _Shape __end, _Ts&&... __values) noexcept(
|
||||
__nothrow_callable<_Fn&, _Shape, decltype(__values)&...>)
|
||||
{
|
||||
for (; __begin != __end; ++__begin)
|
||||
{
|
||||
// Pass a copy of `__begin` to the function so it can't do anything funny with it.
|
||||
__fn_(_Shape(__begin), __values...);
|
||||
}
|
||||
}
|
||||
|
||||
_Fn __fn_;
|
||||
};
|
||||
|
||||
// This function is called when `connect` is called on a `bulk` sender. It transforms
|
||||
// the `bulk` sender into a `bulk_chunked` sender.
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static auto transform_sender(set_value_t, _Sndr&& __sndr, ::cuda::std::__ignore_t)
|
||||
{
|
||||
static_assert(__same_as<tag_of_t<_Sndr>, bulk_t>);
|
||||
auto& [__tag, __data, __child] = __sndr;
|
||||
auto& [__policy, __shape, __fn] = __data;
|
||||
|
||||
using __chunked_fn_t = __bulk_chunked_fn<decltype(__shape), decltype(__fn)>;
|
||||
|
||||
// Lower `bulk` to `bulk_chunked`. If `bulk_chunked` has a late customization, we will
|
||||
// see the customization.
|
||||
return bulk_chunked(::cuda::std::forward_like<_Sndr>(__child),
|
||||
__policy,
|
||||
__shape,
|
||||
__chunked_fn_t{::cuda::std::forward_like<_Sndr>(__fn)});
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr bool __is_chunked() noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto bulk = bulk_t{};
|
||||
|
||||
template <class _Sndr, class _Policy, class _Shape, class _Fn>
|
||||
inline constexpr int structured_binding_size<bulk_t::__sndr_t<_Sndr, _Policy, _Shape, _Fn>> = 3;
|
||||
|
||||
template <class _Sndr, class _Policy, class _Shape, class _Fn>
|
||||
inline constexpr int structured_binding_size<bulk_chunked_t::__sndr_t<_Sndr, _Policy, _Shape, _Fn>> = 3;
|
||||
|
||||
template <class _Sndr, class _Policy, class _Shape, class _Fn>
|
||||
inline constexpr int structured_binding_size<bulk_unchunked_t::__sndr_t<_Sndr, _Policy, _Shape, _Fn>> = 3;
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_BULK
|
||||
@@ -1,193 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_COMPLETION_BEHAVIOR
|
||||
#define __CUDAX_EXECUTION_COMPLETION_BEHAVIOR
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__tuple_dir/ignore.h>
|
||||
#include <cuda/std/__type_traits/integral_constant.h>
|
||||
#include <cuda/std/__type_traits/is_callable.h>
|
||||
#include <cuda/std/__type_traits/is_convertible.h>
|
||||
#include <cuda/std/__utility/rel_ops.h>
|
||||
#include <cuda/std/initializer_list>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// get_completion_behavior
|
||||
namespace __completion_behavior
|
||||
{
|
||||
enum class _CCCL_TYPE_VISIBILITY_DEFAULT completion_behavior : int
|
||||
{
|
||||
unknown, ///< The completion behavior is unknown.
|
||||
asynchronous, ///< The operation's completion will not happen on the calling thread before `start()`
|
||||
///< returns.
|
||||
synchronous, ///< The operation's completion happens-before the return of `start()`.
|
||||
inline_completion ///< The operation completes synchronously within `start()` on the same thread that called
|
||||
///< `start()`.
|
||||
};
|
||||
|
||||
template <completion_behavior _CB>
|
||||
using __constant_t = ::cuda::std::integral_constant<completion_behavior, _CB>;
|
||||
|
||||
using __unknown_t = __constant_t<completion_behavior::unknown>;
|
||||
using __asynchronous_t = __constant_t<completion_behavior::asynchronous>;
|
||||
using __synchronous_t = __constant_t<completion_behavior::synchronous>;
|
||||
using __inline_completion_t = __constant_t<completion_behavior::inline_completion>;
|
||||
} // namespace __completion_behavior
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT min_t;
|
||||
|
||||
struct completion_behavior
|
||||
{
|
||||
private:
|
||||
template <__completion_behavior::completion_behavior _CB>
|
||||
using __constant_t = ::cuda::std::integral_constant<__completion_behavior::completion_behavior, _CB>;
|
||||
|
||||
friend struct min_t;
|
||||
|
||||
public:
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT unknown_t : __completion_behavior::__unknown_t
|
||||
{};
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT asynchronous_t : __completion_behavior::__asynchronous_t
|
||||
{};
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT synchronous_t : __completion_behavior::__synchronous_t
|
||||
{};
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT inline_completion_t : __completion_behavior::__inline_completion_t
|
||||
{};
|
||||
|
||||
static constexpr unknown_t unknown{};
|
||||
static constexpr asynchronous_t asynchronous{};
|
||||
static constexpr synchronous_t synchronous{};
|
||||
static constexpr inline_completion_t inline_completion{};
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// get_completion_behavior: A sender can define this attribute to describe the sender's
|
||||
// completion behavior
|
||||
struct get_completion_behavior_t
|
||||
{
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
operator()(::cuda::std::__ignore_t, ::cuda::std::__ignore_t = {}) const noexcept
|
||||
{
|
||||
return completion_behavior::unknown;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Attrs)
|
||||
_CCCL_REQUIRES(__queryable_with<_Attrs, get_completion_behavior_t>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
operator()(const _Attrs& __attrs, ::cuda::std::__ignore_t = {}) const noexcept
|
||||
{
|
||||
static_assert(__nothrow_queryable_with<_Attrs, get_completion_behavior_t>,
|
||||
"The get_completion_behavior query must be noexcept.");
|
||||
static_assert(::cuda::std::is_convertible_v<__query_result_t<_Attrs, get_completion_behavior_t>,
|
||||
__completion_behavior::completion_behavior>,
|
||||
"The get_completion_behavior query must return one of the static member variables in "
|
||||
"execution::completion_behavior.");
|
||||
return __attrs.query(*this);
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Attrs, class _Env)
|
||||
_CCCL_REQUIRES(__queryable_with<_Attrs, get_completion_behavior_t, const _Env&>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Attrs& __attrs, const _Env& __env) const noexcept
|
||||
{
|
||||
static_assert(__nothrow_queryable_with<_Attrs, get_completion_behavior_t, const _Env&>,
|
||||
"The get_completion_behavior query must be noexcept.");
|
||||
static_assert(::cuda::std::is_convertible_v<__query_result_t<_Attrs, get_completion_behavior_t, const _Env&>,
|
||||
__completion_behavior::completion_behavior>,
|
||||
"The get_completion_behavior query must return one of the static member variables in "
|
||||
"execution::completion_behavior.");
|
||||
return __attrs.query(*this, __env);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto query(forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT min_t
|
||||
{
|
||||
using __completion_behavior_t = __completion_behavior::completion_behavior;
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto
|
||||
__minimum(::cuda::std::initializer_list<__completion_behavior_t> __cbs) noexcept -> __completion_behavior_t
|
||||
{
|
||||
auto __result = __completion_behavior::completion_behavior::inline_completion;
|
||||
for (auto __cb : __cbs)
|
||||
{
|
||||
if (__cb < __result)
|
||||
{
|
||||
__result = __cb;
|
||||
}
|
||||
}
|
||||
return __result;
|
||||
}
|
||||
|
||||
template <__completion_behavior_t... _CBs>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
operator()(completion_behavior::__constant_t<_CBs>...) const noexcept
|
||||
{
|
||||
constexpr auto __behavior = __minimum({_CBs...});
|
||||
|
||||
if constexpr (__behavior == completion_behavior::unknown)
|
||||
{
|
||||
return completion_behavior::unknown;
|
||||
}
|
||||
else if constexpr (__behavior == completion_behavior::asynchronous)
|
||||
{
|
||||
return completion_behavior::asynchronous;
|
||||
}
|
||||
else if constexpr (__behavior == completion_behavior::synchronous)
|
||||
{
|
||||
return completion_behavior::synchronous;
|
||||
}
|
||||
else if constexpr (__behavior == completion_behavior::inline_completion)
|
||||
{
|
||||
return completion_behavior::inline_completion;
|
||||
}
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT min_t min{};
|
||||
|
||||
template <class _Sndr, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_completion_behavior() noexcept
|
||||
{
|
||||
using __behavior_t = __call_result_t<get_completion_behavior_t, env_of_t<_Sndr>, const _Env&...>;
|
||||
return __behavior_t{};
|
||||
}
|
||||
|
||||
template <class _Attrs, class... _Env>
|
||||
_CCCL_CONCEPT __completes_inline =
|
||||
(__call_result_t<get_completion_behavior_t, const _Attrs&, const _Env&...>{}
|
||||
== completion_behavior::inline_completion);
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_COMPLETION_BEHAVIOR
|
||||
@@ -1,663 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_COMPLETION_SIGNATURES
|
||||
#define __CUDAX_EXECUTION_COMPLETION_SIGNATURES
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__type_traits/is_specialization_of.h>
|
||||
#include <cuda/std/__tuple_dir/ignore.h>
|
||||
#include <cuda/std/__type_traits/conditional.h>
|
||||
#include <cuda/std/__type_traits/integral_constant.h>
|
||||
#include <cuda/std/__type_traits/is_empty.h>
|
||||
#include <cuda/std/__type_traits/is_trivially_constructible.h>
|
||||
#include <cuda/std/__type_traits/remove_const.h>
|
||||
#include <cuda/std/__type_traits/type_list.h>
|
||||
#include <cuda/std/__type_traits/type_set.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
|
||||
// include this last:
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wunused-but-set-parameter")
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
using ::cuda::std::__type_list;
|
||||
|
||||
// __partitioned_completions is a cache of completion signatures for fast
|
||||
// access. The completion_signatures<Sigs...>::__partitioned nested struct
|
||||
// inherits from __partitioned_completions. If the cache is never accessed,
|
||||
// it is never instantiated.
|
||||
template <class _ValueTuplesList = __type_list<>, class _ErrorsList = __type_list<>, class _StoppedList = __type_list<>>
|
||||
struct __partitioned_completions;
|
||||
|
||||
template <class... _ValueTuples, class... _Errors, class... _Stopped>
|
||||
struct __partitioned_completions<__type_list<_ValueTuples...>, __type_list<_Errors...>, __type_list<_Stopped...>>
|
||||
{
|
||||
template <template <class...> class _Tuple, template <class...> class _Variant>
|
||||
using __value_types _CCCL_NODEBUG_ALIAS =
|
||||
_Variant<::cuda::std::__type_call1<_ValueTuples, ::cuda::std::__type_quote<_Tuple>>...>;
|
||||
|
||||
template <template <class...> class _Variant, template <class...> class _Transform = ::cuda::std::__type_self_t>
|
||||
using __error_types _CCCL_NODEBUG_ALIAS = _Variant<_Transform<_Errors>...>;
|
||||
|
||||
template <template <class...> class _Variant, class _Type = set_stopped_t()>
|
||||
using __stopped_types _CCCL_NODEBUG_ALIAS = _Variant<__type_second<_Stopped, _Type>...>;
|
||||
|
||||
using __count_values = ::cuda::std::integral_constant<size_t, sizeof...(_ValueTuples)>;
|
||||
using __count_errors = ::cuda::std::integral_constant<size_t, sizeof...(_Errors)>;
|
||||
using __count_stopped = ::cuda::std::integral_constant<size_t, sizeof...(_Stopped)>;
|
||||
|
||||
struct __nothrow_decay_copyable
|
||||
{
|
||||
// These aliases are placed in a separate struct to avoid computing them
|
||||
// if they are not needed.
|
||||
using __fn = ::cuda::std::__type_quote<__nothrow_decay_copyable_t>;
|
||||
using __values = ::cuda::std::_And<::cuda::std::__type_call1<_ValueTuples, __fn>...>;
|
||||
using __errors = __nothrow_decay_copyable_t<_Errors...>;
|
||||
using __all = ::cuda::std::_And<__values, __errors>;
|
||||
};
|
||||
};
|
||||
|
||||
template <class _Tag>
|
||||
struct __partitioned_fold_fn;
|
||||
|
||||
template <>
|
||||
struct __partitioned_fold_fn<set_value_t>
|
||||
{
|
||||
template <class... _ValueTuples, class _Errors, class _Stopped, class _Values>
|
||||
_CCCL_HOST_DEVICE_API auto operator()(__partitioned_completions<__type_list<_ValueTuples...>, _Errors, _Stopped>&,
|
||||
::cuda::std::__undefined<_Values>&) const
|
||||
-> ::cuda::std::__undefined<__partitioned_completions<__type_list<_ValueTuples..., _Values>, _Errors, _Stopped>>&;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct __partitioned_fold_fn<set_error_t>
|
||||
{
|
||||
template <class _Values, class... _Errors, class _Stopped, class _Error>
|
||||
_CCCL_HOST_DEVICE_API auto operator()(__partitioned_completions<_Values, __type_list<_Errors...>, _Stopped>&,
|
||||
::cuda::std::__undefined<__type_list<_Error>>&) const
|
||||
-> ::cuda::std::__undefined<__partitioned_completions<_Values, __type_list<_Errors..., _Error>, _Stopped>>&;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct __partitioned_fold_fn<set_stopped_t>
|
||||
{
|
||||
template <class _Values, class _Errors, class _Stopped>
|
||||
_CCCL_HOST_DEVICE_API auto
|
||||
operator()(__partitioned_completions<_Values, _Errors, _Stopped>&, ::cuda::std::__ignore_t) const
|
||||
-> ::cuda::std::__undefined<__partitioned_completions<_Values, _Errors, __type_list<set_stopped_t()>>>&;
|
||||
};
|
||||
|
||||
// The following overload of binary operator* is used to build up the cache of completion
|
||||
// signatures. We fold over operator*, accumulating the completion signatures in the
|
||||
// cache. `__undefined` is used here to prevent the instantiation of the intermediate
|
||||
// types.
|
||||
template <class _Partitioned, class _Tag, class... _Args>
|
||||
_CCCL_HOST_DEVICE_API auto operator*(::cuda::std::__undefined<_Partitioned>&, _Tag (*)(_Args...)) -> ::cuda::std::
|
||||
__call_result_t<__partitioned_fold_fn<_Tag>, _Partitioned&, ::cuda::std::__undefined<__type_list<_Args...>>&>;
|
||||
|
||||
// This function declaration is used to extract the cache from the `__undefined` type.
|
||||
template <class _Partitioned>
|
||||
_CCCL_HOST_DEVICE_API auto __unpack_partitioned_completions(::cuda::std::__undefined<_Partitioned>&) -> _Partitioned;
|
||||
|
||||
template <class... _Sigs>
|
||||
using __partition_completion_signatures_t _CCCL_NODEBUG_ALIAS = //
|
||||
decltype(execution::__unpack_partitioned_completions(
|
||||
(declval<::cuda::std::__undefined<__partitioned_completions<>>&>() * ... * static_cast<_Sigs*>(nullptr))));
|
||||
|
||||
template <class _Completions>
|
||||
using __partitioned_completions_of_t _CCCL_NODEBUG_ALIAS = typename _Completions::__partitioned::type;
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// completion signatures type traits
|
||||
template <class _Sigs, template <class...> class _Tuple, template <class...> class _Variant>
|
||||
using __value_types _CCCL_NODEBUG_ALIAS =
|
||||
typename __partitioned_completions_of_t<_Sigs>::template __value_types<_Tuple, _Variant>;
|
||||
|
||||
template <class _Sndr, class _Env, template <class...> class _Tuple, template <class...> class _Variant>
|
||||
using value_types_of_t _CCCL_NODEBUG_ALIAS =
|
||||
__value_types<completion_signatures_of_t<_Sndr, _Env>,
|
||||
::cuda::std::__type_indirect_quote<_Tuple>::template __call,
|
||||
::cuda::std::__type_indirect_quote<_Variant>::template __call>;
|
||||
|
||||
template <class _Sigs,
|
||||
template <class...> class _Variant,
|
||||
template <class...> class _Transform = ::cuda::std::__type_self_t>
|
||||
using __error_types _CCCL_NODEBUG_ALIAS =
|
||||
typename __partitioned_completions_of_t<_Sigs>::template __error_types<_Variant, _Transform>;
|
||||
|
||||
template <class _Sndr, class _Env, template <class...> class _Variant>
|
||||
using error_types_of_t _CCCL_NODEBUG_ALIAS =
|
||||
__error_types<completion_signatures_of_t<_Sndr, _Env>, ::cuda::std::__type_indirect_quote<_Variant>::template __call>;
|
||||
|
||||
template <class _Sigs, template <class...> class _Variant, class _Type = set_stopped_t()>
|
||||
using __stopped_types _CCCL_NODEBUG_ALIAS =
|
||||
typename __partitioned_completions_of_t<_Sigs>::template __stopped_types<_Variant, _Type>;
|
||||
|
||||
template <class _Sigs>
|
||||
inline constexpr bool __sends_stopped = __partitioned_completions_of_t<_Sigs>::__count_stopped::value != 0;
|
||||
|
||||
template <class _Sndr, class... _Env>
|
||||
inline constexpr bool sends_stopped = __sends_stopped<completion_signatures_of_t<_Sndr, _Env...>>;
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// __valid_completion_signatures
|
||||
template <class _Ty>
|
||||
_CCCL_CONCEPT __valid_completion_signatures =
|
||||
::cuda::__is_specialization_of_v<::cuda::std::remove_const_t<_Ty>, completion_signatures>;
|
||||
|
||||
template <class... _Sigs>
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL void __assert_valid_completion_signatures(const completion_signatures<_Sigs...>&)
|
||||
{}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// make_completion_signatures
|
||||
template <class _Tag, class... _As>
|
||||
_CCCL_HOST_DEVICE_API auto __normalize_impl(_As&&...) -> _Tag (*)(_As...);
|
||||
|
||||
template <class _Tag, class... _As>
|
||||
_CCCL_HOST_DEVICE_API auto __normalize(_Tag (*)(_As...))
|
||||
-> decltype(execution::__normalize_impl<_Tag>(declval<_As>()...));
|
||||
|
||||
template <class... _Sigs>
|
||||
_CCCL_HOST_DEVICE_API auto __make_unique(_Sigs*...)
|
||||
-> ::cuda::std::__type_apply<::cuda::std::__type_quote<completion_signatures>, ::cuda::std::__make_type_set<_Sigs...>>;
|
||||
|
||||
template <class... _Sigs>
|
||||
using __make_completion_signatures_t _CCCL_NODEBUG_ALIAS =
|
||||
decltype(execution::__make_unique(execution::__normalize(static_cast<_Sigs*>(nullptr))...));
|
||||
|
||||
template <class... _ExplicitSigs, class... _DeducedSigs>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto make_completion_signatures(_DeducedSigs*...) noexcept
|
||||
-> __make_completion_signatures_t<_ExplicitSigs..., _DeducedSigs...>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// concat_completion_signatures
|
||||
struct __concat_completion_signatures_impl;
|
||||
|
||||
template <class... _Sigs>
|
||||
using __concat_completion_signatures_t _CCCL_NODEBUG_ALIAS =
|
||||
__call_result_t<__call_result_t<__concat_completion_signatures_impl, const _Sigs&...>>;
|
||||
|
||||
struct __concat_completion_signatures_fn
|
||||
{
|
||||
template <class... _Sigs>
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto operator()(const _Sigs&...) const noexcept
|
||||
-> __concat_completion_signatures_t<_Sigs...>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
extern const completion_signatures<>& __empty_completion_signatures;
|
||||
|
||||
struct __concat_completion_signatures_impl
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto operator()() const noexcept -> completion_signatures<> (*)()
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
template <class... _Sigs>
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto operator()(const completion_signatures<_Sigs...>&) const noexcept
|
||||
-> __make_completion_signatures_t<_Sigs...> (*)()
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
template <class _Self = __concat_completion_signatures_impl,
|
||||
class... _As,
|
||||
class... _Bs,
|
||||
class... _Cs,
|
||||
class... _Ds,
|
||||
class... _Rest>
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto operator()(
|
||||
const completion_signatures<_As...>&,
|
||||
const completion_signatures<_Bs...>&,
|
||||
const completion_signatures<_Cs...>& = __empty_completion_signatures,
|
||||
const completion_signatures<_Ds...>& = __empty_completion_signatures,
|
||||
const _Rest&...) const noexcept
|
||||
{
|
||||
using _Tmp = completion_signatures<_As..., _Bs..., _Cs..., _Ds...>;
|
||||
using _SigsFnPtr _CCCL_NODEBUG_ALIAS = __call_result_t<_Self, const _Tmp&, const _Rest&...>;
|
||||
return static_cast<_SigsFnPtr>(nullptr);
|
||||
}
|
||||
|
||||
template <class _Ap,
|
||||
class _Bp = ::cuda::std::__ignore_t,
|
||||
class _Cp = ::cuda::std::__ignore_t,
|
||||
class _Dp = ::cuda::std::__ignore_t,
|
||||
class... _Rest>
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto
|
||||
operator()(const _Ap&, const _Bp& = {}, const _Cp& = {}, const _Dp& = {}, const _Rest&...) const noexcept
|
||||
{
|
||||
if constexpr (!__valid_completion_signatures<_Ap>)
|
||||
{
|
||||
return static_cast<_Ap (*)()>(nullptr);
|
||||
}
|
||||
else if constexpr (!__valid_completion_signatures<_Bp>)
|
||||
{
|
||||
return static_cast<_Bp (*)()>(nullptr);
|
||||
}
|
||||
else if constexpr (!__valid_completion_signatures<_Cp>)
|
||||
{
|
||||
return static_cast<_Cp (*)()>(nullptr);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(!__valid_completion_signatures<_Dp>);
|
||||
return static_cast<_Dp (*)()>(nullptr);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT __concat_completion_signatures_fn concat_completion_signatures{};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// implementation details of the completion_signatures class template
|
||||
struct _IN_COMPLETION_SIGNATURES_APPLY;
|
||||
struct _IN_COMPLETION_SIGNATURES_TRANSFORM_REDUCE;
|
||||
struct _FUNCTION_IS_NOT_CALLABLE_WITH_THESE_SIGNATURES;
|
||||
|
||||
template <class... _Sigs>
|
||||
struct __remove_sigs
|
||||
{
|
||||
template <class _Sig>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Sig*) const noexcept -> bool
|
||||
{
|
||||
return !::cuda::std::__type_set_contains_v<::cuda::std::__type_set<_Sigs...>, _Sig>;
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Fn, class _Sig>
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __filer_one() noexcept
|
||||
-> ::cuda::std::_If<_Fn{}(static_cast<_Sig*>(nullptr)), completion_signatures<_Sig>, completion_signatures<>>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
// working around compiler bugs in gcc and msvc
|
||||
template <class... _Sigs>
|
||||
using __completion_signatures = completion_signatures<_Sigs...>;
|
||||
|
||||
template <class... _Values>
|
||||
using __set_value_sig_t = set_value_t(_Values...);
|
||||
|
||||
template <class _Error>
|
||||
using __set_error_sig_t = set_error_t(_Error);
|
||||
|
||||
//! @brief Represents a set of completion signatures for senders in the CUDA C++ execution
|
||||
//! model.
|
||||
//!
|
||||
//! The `completion_signatures` class template is used to describe the possible ways a
|
||||
//! sender may complete. Each signature is a function type of the form
|
||||
//! `set_value_t(Ts...)`, `set_error_t(E)`, or `set_stopped_t()`. This type provides
|
||||
//! compile-time utilities for querying, combining, and transforming sets of completion
|
||||
//! signatures.
|
||||
//!
|
||||
//! @tparam _Sigs... The completion signature types to include in this set.
|
||||
//!
|
||||
//! Example usage:
|
||||
//! @code
|
||||
//! constexpr auto sigs = completion_signatures<set_value_t(int), set_error_t(float), set_stopped_t()>{};
|
||||
//! static_assert(sigs.size() == 3);
|
||||
//! static_assert(sigs.contains<set_value_t(int)>());
|
||||
//! @endcode
|
||||
template <class... _Sigs>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT completion_signatures
|
||||
{
|
||||
//! @brief Partitioned view of the completion signatures for efficient querying.
|
||||
struct __partitioned
|
||||
{
|
||||
// This is defined in a nested struct to avoid computing these types if they are not
|
||||
// needed.
|
||||
using type _CCCL_NODEBUG_ALIAS = __partition_completion_signatures_t<_Sigs...>;
|
||||
};
|
||||
|
||||
//! @brief Type set view of the completion signatures for set operations.
|
||||
struct __type_set
|
||||
{
|
||||
// This is defined in a nested struct to avoid computing this type if it is not
|
||||
// needed.
|
||||
using type _CCCL_NODEBUG_ALIAS = ::cuda::std::__make_type_set<_Sigs...>;
|
||||
};
|
||||
|
||||
//! @brief Applies a metafunction to each signature and collects the results.
|
||||
//! @tparam _Fn The metafunction to apply.
|
||||
//! @tparam _Continuation The template to collect results into.
|
||||
template <template <class...> class _Fn, template <class...> class _Continuation = __completion_signatures>
|
||||
using __transform_q _CCCL_NODEBUG_ALIAS = _Continuation<::cuda::std::__type_apply_q<_Fn, _Sigs>...>;
|
||||
|
||||
//! @brief Applies a callable metafunction to each signature and collects the results.
|
||||
//! @tparam _Fn The callable metafunction to apply.
|
||||
//! @tparam _Continuation The template to collect results into.
|
||||
template <class _Fn, class _Continuation = ::cuda::std::__type_quote<__completion_signatures>>
|
||||
using __transform _CCCL_NODEBUG_ALIAS =
|
||||
::cuda::std::__type_call<_Continuation, ::cuda::std::__type_apply<_Fn, _Sigs>...>;
|
||||
|
||||
//! @brief Calls a metafunction with the signatures as arguments.
|
||||
//! @tparam _Fn The metafunction to call.
|
||||
//! @tparam _More Additional arguments to pass.
|
||||
template <class _Fn, class... _More>
|
||||
using __call _CCCL_NODEBUG_ALIAS = ::cuda::std::__type_call<_Fn, _More..., _Sigs...>;
|
||||
|
||||
//! @brief Default constructor.
|
||||
_CCCL_HIDE_FROM_ABI constexpr completion_signatures() = default;
|
||||
|
||||
//! @brief Returns the number of completion signatures in the set.
|
||||
//! @return The number of signatures.
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto size() noexcept -> size_t
|
||||
{
|
||||
return sizeof...(_Sigs);
|
||||
}
|
||||
|
||||
//! @brief Counts the number of signatures with the given tag.
|
||||
//! @tparam _Tag The tag to count (e.g., set_value, set_error, set_stopped).
|
||||
//! @return The number of signatures with the given tag.
|
||||
template <class _Tag>
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto count(_Tag) noexcept -> size_t
|
||||
{
|
||||
if constexpr (_Tag{} == set_value)
|
||||
{
|
||||
return __partitioned::type::__count_values::value;
|
||||
}
|
||||
else if constexpr (_Tag{} == set_error)
|
||||
{
|
||||
return __partitioned::type::__count_errors::value;
|
||||
}
|
||||
else
|
||||
{
|
||||
return __partitioned::type::__count_stopped::value;
|
||||
}
|
||||
}
|
||||
|
||||
//! @brief Checks if the set contains the given signature.
|
||||
//! @tparam _Sig The signature type to check.
|
||||
//! @return true if the signature is present, false otherwise.
|
||||
template <class _Sig>
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto contains(_Sig* = nullptr) noexcept -> bool
|
||||
{
|
||||
return ::cuda::std::__type_set_contains_v<typename __type_set::type, _Sig>;
|
||||
}
|
||||
|
||||
//! @brief Applies a callable to all signatures in the set.
|
||||
//! @tparam _Fn The callable to apply.
|
||||
//! @param __fn The callable instance.
|
||||
//! @return The result of calling __fn with all signatures as arguments.
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Fn>
|
||||
_CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto apply(_Fn __fn) -> __call_result_t<_Fn, _Sigs*...>
|
||||
{
|
||||
return __fn(static_cast<_Sigs*>(nullptr)...);
|
||||
}
|
||||
|
||||
//! @brief Filters the set using a predicate, returning a new set with only matching
|
||||
//! signatures.
|
||||
//! @tparam _Fn The predicate type (must be empty and trivially constructible).
|
||||
//! @param The predicate instance.
|
||||
//! @return A new completion_signatures set with only the signatures for which the
|
||||
//! predicate returns true.
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Fn>
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto filter(_Fn)
|
||||
{
|
||||
static_assert(::cuda::std::is_empty_v<_Fn> && ::cuda::std::is_trivially_constructible_v<_Fn>,
|
||||
"The filter function must be empty and trivially constructible.");
|
||||
return concat_completion_signatures(execution::__filer_one<_Fn, _Sigs>()...);
|
||||
}
|
||||
|
||||
//! @brief Selects all signatures with the given tag.
|
||||
//! @tparam _Tag The tag to select (e.g., set_value, set_error, set_stopped).
|
||||
//! @return A new completion_signatures set containing only signatures with the given
|
||||
//! tag.
|
||||
template <class _Tag>
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto select(_Tag) noexcept
|
||||
{
|
||||
if constexpr (_Tag{} == set_value)
|
||||
{
|
||||
return __value_types<completion_signatures, __set_value_sig_t, __completion_signatures>{};
|
||||
}
|
||||
else if constexpr (_Tag{} == set_error)
|
||||
{
|
||||
return __error_types<completion_signatures, __completion_signatures, __set_error_sig_t>{};
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(_Tag{} == set_stopped, "The tag must be set_value, set_error, or set_stopped.");
|
||||
return __stopped_types<completion_signatures, __completion_signatures>{};
|
||||
}
|
||||
}
|
||||
|
||||
//! @brief Applies a transform and then reduces the results.
|
||||
//! @tparam _Transform The transform callable.
|
||||
//! @tparam _Reduce The reduce callable.
|
||||
//! @param __transform The transform instance.
|
||||
//! @param __reduce The reduce instance.
|
||||
//! @return The result of reducing the transformed signatures.
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Transform, class _Reduce>
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto transform_reduce(_Transform __transform, _Reduce __reduce)
|
||||
-> __call_result_t<_Reduce, __call_result_t<_Transform, _Sigs*>...>
|
||||
{
|
||||
return __reduce(__transform(static_cast<_Sigs*>(nullptr))...);
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES completion_signatures() -> completion_signatures<>;
|
||||
|
||||
// work-around for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95629
|
||||
#if _CCCL_COMPILER(GCC, ==, 11)
|
||||
# define _CCCL_CONSTEVAL_OPERATOR constexpr
|
||||
#else // ^^^ GCC 11 ^^^ / vvv other compilers vvv
|
||||
# define _CCCL_CONSTEVAL_OPERATOR _CCCL_CONSTEVAL
|
||||
#endif // ^^^ other compilers ^^^
|
||||
|
||||
//! @brief Returns the union of two sets of completion signatures.
|
||||
//! @tparam _SelfSigs The first set of signature types.
|
||||
//! @tparam _OtherSigs The other set of signature types.
|
||||
//! @param __self The first `completion_signatures` object.
|
||||
//! @param __other The other `completion_signatures` object.
|
||||
//! @return The union of the two sets.
|
||||
template <class... _SelfSigs, class... _OtherSigs>
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL_OPERATOR auto
|
||||
operator+([[maybe_unused]] completion_signatures<_SelfSigs...> __self,
|
||||
[[maybe_unused]] completion_signatures<_OtherSigs...> __other) noexcept
|
||||
{
|
||||
if constexpr (sizeof...(_SelfSigs) == 0) // short-circuit some common cases
|
||||
{
|
||||
return __other;
|
||||
}
|
||||
else if constexpr (sizeof...(_OtherSigs) == 0)
|
||||
{
|
||||
return __self;
|
||||
}
|
||||
else
|
||||
{
|
||||
return concat_completion_signatures(__self, __other);
|
||||
}
|
||||
}
|
||||
|
||||
//! @brief Returns the set difference between two sets of completion signatures.
|
||||
//! @tparam _SelfSigs The first set of signature types.
|
||||
//! @tparam _OtherSigs The second set of signature types.
|
||||
//! @return A new set with all signatures from the other set removed.
|
||||
template <class... _SelfSigs, class... _OtherSigs>
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL_OPERATOR auto
|
||||
operator-(completion_signatures<_SelfSigs...> __self, completion_signatures<_OtherSigs...>) noexcept
|
||||
{
|
||||
if constexpr (sizeof...(_OtherSigs) == 0 || sizeof...(_SelfSigs) == 0) // short-circuit some common cases
|
||||
{
|
||||
return __self;
|
||||
}
|
||||
else
|
||||
{
|
||||
return __self.filter(__remove_sigs<_OtherSigs...>{});
|
||||
}
|
||||
}
|
||||
|
||||
//! @brief Checks if two completion_signatures sets are equal.
|
||||
//! @tparam _SelfSigs The first set of signature types.
|
||||
//! @tparam _OtherSigs The second set of signature types.
|
||||
//! @return `true` if the sets are equal, `false` otherwise.
|
||||
template <class... _SelfSigs, class... _OtherSigs>
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL_OPERATOR auto
|
||||
operator==(completion_signatures<_SelfSigs...>, completion_signatures<_OtherSigs...>) noexcept -> bool
|
||||
{
|
||||
if constexpr (sizeof...(_OtherSigs) != sizeof...(_SelfSigs))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
else
|
||||
{
|
||||
using __signatures_set_t = typename completion_signatures<_SelfSigs...>::__type_set::type;
|
||||
return ::cuda::std::__type_set_contains_v<__signatures_set_t, _OtherSigs...>;
|
||||
}
|
||||
}
|
||||
|
||||
//! @brief Checks if two completion_signatures sets are not equal.
|
||||
//! @tparam _SelfSigs The first set of signature types.
|
||||
//! @tparam _OtherSigs The second set of signature types.
|
||||
//! @param __self The other `completion_signatures` object.
|
||||
//! @param __other The other `completion_signatures` object.
|
||||
//! @return `true` if the sets are not equal, `false` otherwise.
|
||||
template <class... _SelfSigs, class... _OtherSigs>
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL_OPERATOR auto
|
||||
operator!=(completion_signatures<_SelfSigs...> __self, completion_signatures<_OtherSigs...> __other) noexcept -> bool
|
||||
{
|
||||
return !(__self == __other);
|
||||
}
|
||||
|
||||
#undef _CCCL_CONSTEVAL_OPERATOR
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// __gather_completion_signatures
|
||||
template <class _WantedTag>
|
||||
struct __gather_sigs_fn;
|
||||
|
||||
template <>
|
||||
struct __gather_sigs_fn<set_value_t>
|
||||
{
|
||||
template <class _Sigs, template <class...> class _Tuple, template <class...> class _Variant>
|
||||
using __call _CCCL_NODEBUG_ALIAS = __value_types<_Sigs, _Tuple, _Variant>;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct __gather_sigs_fn<set_error_t>
|
||||
{
|
||||
template <class _Sigs, template <class...> class _Tuple, template <class...> class _Variant>
|
||||
using __call _CCCL_NODEBUG_ALIAS = __error_types<_Sigs, _Variant, _Tuple>;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct __gather_sigs_fn<set_stopped_t>
|
||||
{
|
||||
template <class _Sigs, template <class...> class _Tuple, template <class...> class _Variant>
|
||||
using __call _CCCL_NODEBUG_ALIAS = __stopped_types<_Sigs, _Variant, _Tuple<>>;
|
||||
};
|
||||
|
||||
template <class _Sigs, class _WantedTag, template <class...> class _Tuple, template <class...> class _Variant>
|
||||
using __gather_completion_signatures _CCCL_NODEBUG_ALIAS =
|
||||
typename __gather_sigs_fn<_WantedTag>::template __call<_Sigs, _Tuple, _Variant>;
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// __eptr_completion and __eptr_completion_if
|
||||
#if _CCCL_HAS_EXCEPTIONS()
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API inline _CCCL_CONSTEVAL auto __eptr_completion() noexcept
|
||||
{
|
||||
return completion_signatures<set_error_t(exception_ptr)>{};
|
||||
}
|
||||
#else // ^^^ _CCCL_HAS_EXCEPTIONS() ^^^ / vvv !_CCCL_HAS_EXCEPTIONS() vvv
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API inline _CCCL_CONSTEVAL auto __eptr_completion() noexcept
|
||||
{
|
||||
return completion_signatures{};
|
||||
}
|
||||
#endif // !_CCCL_HAS_EXCEPTIONS()
|
||||
|
||||
template <bool _PotentiallyThrowing>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __eptr_completion_if() noexcept
|
||||
{
|
||||
if constexpr (_PotentiallyThrowing)
|
||||
{
|
||||
return __eptr_completion();
|
||||
}
|
||||
else
|
||||
{
|
||||
return completion_signatures{};
|
||||
}
|
||||
}
|
||||
|
||||
using __eptr_completion_t _CCCL_NODEBUG_ALIAS = decltype(execution::__eptr_completion());
|
||||
|
||||
template <bool _PotentiallyThrowing>
|
||||
using __eptr_completion_if_t _CCCL_NODEBUG_ALIAS = decltype(execution::__eptr_completion_if<_PotentiallyThrowing>());
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// invalid_completion_signature
|
||||
#if _CCCL_HAS_CONSTEXPR_EXCEPTIONS()
|
||||
|
||||
template <class... _What, class... _Values>
|
||||
[[noreturn, nodiscard]] _CCCL_HOST_DEVICE_API consteval auto invalid_completion_signature(_Values... __values)
|
||||
-> completion_signatures<>
|
||||
{
|
||||
if constexpr (sizeof...(_Values) == 1)
|
||||
{
|
||||
throw __sender_type_check_failure<_Values..., _What...>(__values...);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw __sender_type_check_failure<::cuda::std::__tuple<_Values...>, _What...>(::cuda::std::__tuple{__values...});
|
||||
}
|
||||
}
|
||||
|
||||
#else // ^^^ _CCCL_HAS_CONSTEXPR_EXCEPTIONS() ^^^ / vvv !_CCCL_HAS_CONSTEXPR_EXCEPTIONS() vvv
|
||||
|
||||
template <class... _What, class... _Values>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto invalid_completion_signature(_Values...)
|
||||
{
|
||||
return _ERROR<_What...>{};
|
||||
}
|
||||
|
||||
#endif // ^^^ !_CCCL_HAS_CONSTEXPR_EXCEPTIONS() ^^^
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // _CUDAX_EXECUTION_COMPLETION_SIGNATURES_H
|
||||
@@ -1,160 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_CONCEPTS
|
||||
#define __CUDAX_EXECUTION_CONCEPTS
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__cccl/unreachable.h>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__concepts/constructible.h>
|
||||
#include <cuda/std/__concepts/copyable.h>
|
||||
#include <cuda/std/__concepts/equality_comparable.h>
|
||||
#include <cuda/std/__type_traits/decay.h>
|
||||
#include <cuda/std/__type_traits/integral_constant.h>
|
||||
#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/get_completion_signatures.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
// Receiver concepts:
|
||||
template <class _Rcvr>
|
||||
_CCCL_CONCEPT receiver = //
|
||||
_CCCL_REQUIRES_EXPR((_Rcvr)) //
|
||||
( //
|
||||
requires(__is_receiver<decay_t<_Rcvr>>), //
|
||||
requires(::cuda::std::move_constructible<decay_t<_Rcvr>>), //
|
||||
requires(::cuda::std::constructible_from<decay_t<_Rcvr>, _Rcvr>), //
|
||||
requires(__nothrow_movable<decay_t<_Rcvr>>) //
|
||||
);
|
||||
|
||||
template <class _Rcvr, class _Sig>
|
||||
inline constexpr bool __valid_completion_for = false;
|
||||
|
||||
template <class _Rcvr, class _Tag, class... _As>
|
||||
inline constexpr bool __valid_completion_for<_Rcvr, _Tag(_As...)> = __callable<_Tag, _Rcvr, _As...>;
|
||||
|
||||
template <class _Rcvr, class _Completions>
|
||||
inline constexpr bool __has_completions = false;
|
||||
|
||||
template <class _Rcvr, class... _Sigs>
|
||||
inline constexpr bool __has_completions<_Rcvr, completion_signatures<_Sigs...>> =
|
||||
(__valid_completion_for<_Rcvr, _Sigs> && ...);
|
||||
|
||||
template <class _Rcvr, class _Completions>
|
||||
_CCCL_CONCEPT receiver_of = //
|
||||
_CCCL_REQUIRES_EXPR((_Rcvr, _Completions)) //
|
||||
( //
|
||||
requires(receiver<_Rcvr>), //
|
||||
requires(__has_completions<decay_t<_Rcvr>, _Completions>) //
|
||||
);
|
||||
|
||||
// Queryable traits:
|
||||
template <class _Ty>
|
||||
_CCCL_CONCEPT __queryable = ::cuda::std::destructible<_Ty>;
|
||||
|
||||
// Awaitable traits:
|
||||
template <class>
|
||||
_CCCL_CONCEPT __is_awaitable = false; // TODO: Implement this concept.
|
||||
|
||||
// Sender traits:
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto __enable_sender() -> bool
|
||||
{
|
||||
if constexpr (__is_sender<_Sndr>)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return __is_awaitable<_Sndr>;
|
||||
}
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
inline constexpr bool enable_sender = __enable_sender<_Sndr>();
|
||||
|
||||
// Sender concepts:
|
||||
template <class... _Env>
|
||||
struct __completions_tester
|
||||
{
|
||||
template <class _Sndr, bool _EnableIfConstexpr = ((void) execution::get_completion_signatures<_Sndr, _Env...>(), true)>
|
||||
_CCCL_HOST_DEVICE_API static constexpr auto __is_valid(int) -> bool
|
||||
{
|
||||
return __valid_completion_signatures<completion_signatures_of_t<_Sndr, _Env...>>;
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API static constexpr auto __is_valid(long) -> bool
|
||||
{
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Sndr, class... _Env>
|
||||
_CCCL_CONCEPT __has_valid_completion_signatures = __completions_tester<_Env...>::template __is_valid<_Sndr>(0);
|
||||
|
||||
template <class _Sndr>
|
||||
_CCCL_CONCEPT sender = //
|
||||
_CCCL_REQUIRES_EXPR((_Sndr)) //
|
||||
( //
|
||||
requires(enable_sender<decay_t<_Sndr>>), //
|
||||
requires(::cuda::std::move_constructible<decay_t<_Sndr>>), //
|
||||
requires(::cuda::std::constructible_from<decay_t<_Sndr>, _Sndr>) //
|
||||
);
|
||||
|
||||
template <class _Sndr, class... _Env>
|
||||
_CCCL_CONCEPT sender_in = //
|
||||
_CCCL_REQUIRES_EXPR((_Sndr, variadic _Env)) //
|
||||
( //
|
||||
requires(sender<_Sndr>), //
|
||||
requires(sizeof...(_Env) <= 1), //
|
||||
requires((__queryable<_Env> && ... && true)), //
|
||||
requires(__has_valid_completion_signatures<_Sndr, _Env...>) //
|
||||
);
|
||||
|
||||
template <class _Sndr>
|
||||
_CCCL_CONCEPT dependent_sender = //
|
||||
_CCCL_REQUIRES_EXPR((_Sndr)) //
|
||||
( //
|
||||
requires(sender<_Sndr>), //
|
||||
requires(__is_dependent_sender<_Sndr>()) //
|
||||
);
|
||||
|
||||
// Scheduler concepts:
|
||||
template <class _Sch>
|
||||
_CCCL_CONCEPT scheduler = //
|
||||
_CCCL_REQUIRES_EXPR((_Sch), __declfn_t<_Sch> __sch) //
|
||||
( //
|
||||
requires(__is_scheduler<_Sch>), //
|
||||
schedule(__sch()), //
|
||||
requires(::cuda::std::equality_comparable<::cuda::std::remove_cvref_t<_Sch>>), //
|
||||
requires(::cuda::std::copyable<::cuda::std::remove_cvref_t<_Sch>>) //
|
||||
);
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_CONCEPTS
|
||||
@@ -1,316 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_CONDITIONAL
|
||||
#define __CUDAX_EXECUTION_CONDITIONAL
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__cccl/unreachable.h>
|
||||
#include <cuda/std/__exception/exception_macros.h>
|
||||
#include <cuda/std/__type_traits/is_callable.h>
|
||||
#include <cuda/std/__type_traits/is_convertible.h>
|
||||
#include <cuda/std/__type_traits/type_list.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/concepts.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/exception.cuh>
|
||||
#include <cuda/experimental/__execution/just_from.cuh>
|
||||
#include <cuda/experimental/__execution/meta.cuh>
|
||||
#include <cuda/experimental/__execution/rcvr_ref.cuh>
|
||||
#include <cuda/experimental/__execution/transform_sender.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__execution/variant.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
//! @file conditional.cuh
|
||||
//! This file defines the @c conditional sender. @c conditional is a sender that
|
||||
//! selects between two continuations based on the result of a predecessor. It
|
||||
//! accepts a predecessor, a predicate, and two continuations. It passes the
|
||||
//! result of the predecessor to the predicate. If the predicate returns @c true,
|
||||
//! the result is passed to the first continuation; otherwise, it is passed to
|
||||
//! the second continuation.
|
||||
//!
|
||||
//! By "continuation", we mean a so-called sender adaptor closure: a unary function
|
||||
//! that takes a sender and returns a new sender. The expression `then(f)` is an
|
||||
//! example of a continuation.
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
struct _FUNCTION_MUST_RETURN_A_BOOLEAN_TESTABLE_VALUE;
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT conditional_t
|
||||
{
|
||||
_CUDAX_SEMI_PRIVATE :
|
||||
template <class _Pred, class _Then, class _Else>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_base_t;
|
||||
|
||||
template <class... _As>
|
||||
_CCCL_HOST_DEVICE _CCCL_FORCEINLINE static auto __mk_complete_fn(_As&&... __as) noexcept
|
||||
{
|
||||
return [&](auto __sink) noexcept {
|
||||
return __sink(static_cast<_As&&>(__as)...);
|
||||
};
|
||||
}
|
||||
|
||||
template <class... _As>
|
||||
using __just_from_t _CCCL_NODEBUG_ALIAS = decltype(just_from(conditional_t::__mk_complete_fn(declval<_As>()...)));
|
||||
|
||||
template <class _Pred, class _Then, class _Else, class... _Env>
|
||||
struct __either_sig_fn
|
||||
{
|
||||
template <class... _As>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto operator()() const
|
||||
{
|
||||
if constexpr (!__callable<_Pred, _As&...>)
|
||||
{
|
||||
return invalid_completion_signature<_WHERE(_IN_ALGORITHM, conditional_t),
|
||||
_WHAT(_FUNCTION_IS_NOT_CALLABLE),
|
||||
_WITH_FUNCTION(_Pred),
|
||||
_WITH_ARGUMENTS(_As & ...)>();
|
||||
}
|
||||
else if constexpr (!::cuda::std::is_convertible_v<__call_result_t<_Pred, _As&...>, bool>)
|
||||
{
|
||||
return invalid_completion_signature<_WHERE(_IN_ALGORITHM, conditional_t),
|
||||
_WHAT(_FUNCTION_MUST_RETURN_A_BOOLEAN_TESTABLE_VALUE),
|
||||
_WITH_FUNCTION(_Pred),
|
||||
_WITH_ARGUMENTS(_As & ...)>();
|
||||
}
|
||||
else
|
||||
{
|
||||
return concat_completion_signatures(
|
||||
get_completion_signatures<__call_result_t<_Then, __just_from_t<_As...>>, _Env...>(),
|
||||
get_completion_signatures<__call_result_t<_Else, __just_from_t<_As...>>, _Env...>());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Rcvr, class _Pred, class _Then, class _Else, class _Completions>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __state_t
|
||||
{
|
||||
using __params_t = __closure_base_t<_Pred, _Then, _Else>;
|
||||
|
||||
template <class... _As>
|
||||
using __opstate_list_t =
|
||||
::cuda::std::__type_list<connect_result_t<__call_result_t<_Then, __just_from_t<_As...>>, __rcvr_ref_t<_Rcvr>>,
|
||||
connect_result_t<__call_result_t<_Else, __just_from_t<_As...>>, __rcvr_ref_t<_Rcvr>>>;
|
||||
|
||||
using __next_ops_variant_t _CCCL_NODEBUG_ALIAS =
|
||||
__value_types<_Completions, __opstate_list_t, __type_concat_into_quote<__variant>::__call>;
|
||||
|
||||
_Rcvr __rcvr_;
|
||||
__params_t __params_;
|
||||
__next_ops_variant_t __ops_{};
|
||||
};
|
||||
|
||||
template <class _Rcvr, class _Pred, class _Then, class _Else, class _Completions>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_t
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class... _As>
|
||||
_CCCL_HOST_DEVICE_API void set_value(_As&&... __as) noexcept
|
||||
{
|
||||
auto __just = just_from(conditional_t::__mk_complete_fn(static_cast<_As&&>(__as)...));
|
||||
_CCCL_TRY
|
||||
{
|
||||
if (static_cast<_Pred&&>(__state_->__params_.pred)(__as...))
|
||||
{
|
||||
auto& __op = __state_->__ops_.__emplace_from(
|
||||
connect, static_cast<_Then&&>(__state_->__params_.on_true)(__just), __ref_rcvr(__state_->__rcvr_));
|
||||
execution::start(__op);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto& __op = __state_->__ops_.__emplace_from(
|
||||
connect, static_cast<_Else&&>(__state_->__params_.on_false)(__just), __ref_rcvr(__state_->__rcvr_));
|
||||
execution::start(__op);
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_ALL
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(__state_->__rcvr_), execution::current_exception());
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_error(_Error&& __error) noexcept
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(__state_->__rcvr_), static_cast<_Error&&>(__error));
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_stopped() noexcept
|
||||
{
|
||||
execution::set_stopped(static_cast<_Rcvr&&>(__state_->__rcvr_));
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __fwd_env_t<env_of_t<_Rcvr>>
|
||||
{
|
||||
return __fwd_env(execution::get_env(__state_->__rcvr_));
|
||||
}
|
||||
|
||||
__state_t<_Rcvr, _Pred, _Then, _Else, _Completions>* __state_;
|
||||
};
|
||||
|
||||
template <class _CvSndr, class _Rcvr, class _Pred, class _Then, class _Else>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
using __completions_t = completion_signatures_of_t<_CvSndr, __fwd_env_t<env_of_t<_Rcvr>>>;
|
||||
using __params_t = __closure_base_t<_Pred, _Then, _Else>;
|
||||
using __rcvr_t = conditional_t::__rcvr_t<_Rcvr, _Pred, _Then, _Else, __completions_t>;
|
||||
|
||||
_CCCL_HOST_DEVICE_API __opstate_t(_CvSndr&& __sndr, _Rcvr&& __rcvr, __params_t&& __params)
|
||||
: __state_{static_cast<_Rcvr&&>(__rcvr), static_cast<__params_t&&>(__params)}
|
||||
, __op_{execution::connect(static_cast<_CvSndr&&>(__sndr), __rcvr_t{&__state_})}
|
||||
{}
|
||||
|
||||
_CCCL_IMMOVABLE(__opstate_t);
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void start() noexcept
|
||||
{
|
||||
execution::start(__op_);
|
||||
}
|
||||
|
||||
__state_t<_Rcvr, _Pred, _Then, _Else, __completions_t> __state_;
|
||||
connect_result_t<_CvSndr, __rcvr_t> __op_;
|
||||
};
|
||||
|
||||
public:
|
||||
template <class _Pred, class _Then, class _Else>
|
||||
using params _CCCL_NODEBUG_ALIAS = __closure_base_t<_Pred, _Then, _Else>;
|
||||
|
||||
template <class _Params, class _Sndr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
template <class _Sndr, class _Pred, class _Then, class _Else>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr __sndr, _Pred __pred, _Then __then, _Else __else) const;
|
||||
|
||||
template <class _Pred, class _Then, class _Else>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Pred __pred, _Then __then, _Else __else) const;
|
||||
};
|
||||
|
||||
template <class _Pred, class _Then, class _Else, class _Sndr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT conditional_t::__sndr_t<conditional_t::__closure_base_t<_Pred, _Then, _Else>, _Sndr>
|
||||
{
|
||||
using __params_t _CCCL_NODEBUG_ALIAS = conditional_t::__closure_base_t<_Pred, _Then, _Else>;
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ conditional_t __tag_;
|
||||
__params_t __params_;
|
||||
_Sndr __sndr_;
|
||||
|
||||
template <class _Self, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
_CUDAX_LET_COMPLETIONS(auto(__child_completions) = get_child_completion_signatures<_Self, _Sndr, _Env...>())
|
||||
{
|
||||
return concat_completion_signatures(
|
||||
transform_completion_signatures(__child_completions, __either_sig_fn<_Pred, _Then, _Else, _Env...>{}),
|
||||
__eptr_completion());
|
||||
}
|
||||
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
connect(_Rcvr __rcvr) && -> __opstate_t<_Sndr, _Rcvr, _Pred, _Then, _Else>
|
||||
{
|
||||
return {static_cast<_Sndr&&>(__sndr_), static_cast<_Rcvr&&>(__rcvr), static_cast<__params_t&&>(__params_)};
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
connect(_Rcvr __rcvr) const& -> __opstate_t<_Sndr const&, _Rcvr, _Pred, _Then, _Else>
|
||||
{
|
||||
return {__sndr_, static_cast<_Rcvr&&>(__rcvr), static_cast<__params_t&&>(__params_)};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __fwd_env_t<env_of_t<_Sndr>>
|
||||
{
|
||||
return __fwd_env(execution::get_env(__sndr_));
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Pred, class _Then, class _Else>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT conditional_t::__closure_base_t
|
||||
{
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API auto operator()(_Sndr __sndr) &&
|
||||
{
|
||||
using __sndr_t = conditional_t::__sndr_t<__closure_base_t, _Sndr>;
|
||||
|
||||
// If the incoming sender is non-dependent, we can check the completion signatures of
|
||||
// the composed sender immediately.
|
||||
if constexpr (!dependent_sender<_Sndr>)
|
||||
{
|
||||
__assert_valid_completion_signatures(execution::get_completion_signatures<__sndr_t>());
|
||||
}
|
||||
|
||||
return __sndr_t{{}, static_cast<__closure_base_t&&>(*this), static_cast<_Sndr&&>(__sndr)};
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API auto operator()(_Sndr __sndr) const&
|
||||
{
|
||||
return __closure_base_t(*this)(static_cast<_Sndr&&>(__sndr));
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API friend auto operator|(_Sndr __sndr, __closure_base_t __self)
|
||||
{
|
||||
return static_cast<__closure_base_t&&>(__self)(static_cast<_Sndr&&>(__sndr));
|
||||
}
|
||||
|
||||
_Pred pred;
|
||||
_Then on_true;
|
||||
_Else on_false;
|
||||
};
|
||||
|
||||
template <class _Sndr, class _Pred, class _Then, class _Else>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto
|
||||
conditional_t::operator()(_Sndr __sndr, _Pred __pred, _Then __then, _Else __else) const
|
||||
{
|
||||
using __params_t _CCCL_NODEBUG_ALIAS = __closure_base_t<_Pred, _Then, _Else>;
|
||||
__params_t __params{static_cast<_Pred&&>(__pred), static_cast<_Then&&>(__then), static_cast<_Else&&>(__else)};
|
||||
return static_cast<__params_t&&>(__params)(static_cast<_Sndr&&>(__sndr));
|
||||
}
|
||||
|
||||
template <class _Pred, class _Then, class _Else>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto conditional_t::operator()(_Pred __pred, _Then __then, _Else __else) const
|
||||
{
|
||||
return __closure_base_t<_Pred, _Then, _Else>{
|
||||
static_cast<_Pred&&>(__pred), static_cast<_Then&&>(__then), static_cast<_Else&&>(__else)};
|
||||
}
|
||||
|
||||
template <class _Params, class _Sndr>
|
||||
inline constexpr int structured_binding_size<conditional_t::__sndr_t<_Params, _Sndr>> = 3;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT conditional_t conditional{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_CONDITIONAL
|
||||
@@ -1,481 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_CONTINUES_ON
|
||||
#define __CUDAX_EXECUTION_CONTINUES_ON
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__cccl/unreachable.h>
|
||||
#include <cuda/std/__exception/exception_macros.h>
|
||||
#include <cuda/std/__type_traits/conditional.h>
|
||||
#include <cuda/std/__utility/pod_tuple.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/concepts.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/exception.cuh>
|
||||
#include <cuda/experimental/__execution/get_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/meta.cuh>
|
||||
#include <cuda/experimental/__execution/queries.cuh>
|
||||
#include <cuda/experimental/__execution/rcvr_ref.cuh>
|
||||
#include <cuda/experimental/__execution/schedule_from.cuh>
|
||||
#include <cuda/experimental/__execution/transform_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/transform_sender.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__execution/variant.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
namespace __detail
|
||||
{
|
||||
template <class _Tag>
|
||||
struct __decay_args
|
||||
{
|
||||
template <class... _Ts>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto operator()() const noexcept
|
||||
{
|
||||
if constexpr (!__decay_copyable<_Ts...>)
|
||||
{
|
||||
return invalid_completion_signature<_WHERE(_IN_ALGORITHM, continues_on_t),
|
||||
_WHAT(_ARGUMENTS_ARE_NOT_DECAY_COPYABLE),
|
||||
_WITH_ARGUMENTS(_Ts...)>();
|
||||
}
|
||||
else if constexpr (!__nothrow_decay_copyable<_Ts...>)
|
||||
{
|
||||
return completion_signatures<_Tag(decay_t<_Ts>...), set_error_t(exception_ptr)>{};
|
||||
}
|
||||
else
|
||||
{
|
||||
return completion_signatures<_Tag(decay_t<_Ts>...)>{};
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace __detail
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT continues_on_t
|
||||
{
|
||||
_CUDAX_SEMI_PRIVATE :
|
||||
struct __send_result_fn
|
||||
{
|
||||
template <class _Rcvr, class _Tag, class... _As>
|
||||
_CCCL_HOST_DEVICE_API constexpr void operator()(_Rcvr& __rcvr, _Tag, _As&... __args) const noexcept
|
||||
{
|
||||
// moves from lvalues here is intentional:
|
||||
_Tag{}(static_cast<_Rcvr&&>(__rcvr), static_cast<_As&&>(__args)...);
|
||||
}
|
||||
};
|
||||
|
||||
struct __send_result_visitor
|
||||
{
|
||||
template <class _Rcvr, class _Tuple>
|
||||
_CCCL_HOST_DEVICE_API constexpr void operator()(_Rcvr& __rcvr, _Tuple& __tuple) const noexcept
|
||||
{
|
||||
::cuda::std::__apply(__send_result_fn{}, __tuple, __rcvr);
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Rcvr, class _Results>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __state_base_t
|
||||
{
|
||||
_Rcvr __rcvr_;
|
||||
_Results __result_;
|
||||
};
|
||||
|
||||
// This receiver is connected to the scheduler. It forwards the results of the child sender,
|
||||
// which are stored in a variant, to the parent receiver.
|
||||
template <class _Rcvr, class _Results>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_t
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_value() noexcept
|
||||
{
|
||||
__visit(__send_result_visitor{}, __state_->__result_, __state_->__rcvr_);
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_error(_Error&& __error) noexcept
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(__state_->__rcvr_), static_cast<_Error&&>(__error));
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_stopped() noexcept
|
||||
{
|
||||
execution::set_stopped(static_cast<_Rcvr&&>(__state_->__rcvr_));
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __fwd_env_t<env_of_t<_Rcvr>>
|
||||
{
|
||||
return __fwd_env(execution::get_env(__state_->__rcvr_));
|
||||
}
|
||||
|
||||
__state_base_t<_Rcvr, _Results>* __state_;
|
||||
};
|
||||
|
||||
template <class _Sch, class _Rcvr, class _Results>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __state_t : __state_base_t<_Rcvr, _Results>
|
||||
{
|
||||
connect_result_t<schedule_result_t<_Sch>, __rcvr_t<_Rcvr, _Results>> __opstate2_;
|
||||
};
|
||||
|
||||
// This receiver is connected to the child sender. It stashes the sender's results into
|
||||
// a variant.
|
||||
template <class _Sch, class _Rcvr, class _Results>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __stash_rcvr_t
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
template <class _Tag, class... _As>
|
||||
_CCCL_HOST_DEVICE_API void __set_result(_Tag, _As&&... __as) noexcept
|
||||
{
|
||||
using __tupl_t _CCCL_NODEBUG_ALIAS = ::cuda::std::__tuple<_Tag, decay_t<_As>...>;
|
||||
_CCCL_TRY
|
||||
{
|
||||
__state_->__result_.template __emplace<__tupl_t>(_Tag{}, static_cast<_As&&>(__as)...);
|
||||
}
|
||||
_CCCL_CATCH_ALL
|
||||
{
|
||||
// Avoid ODR-using this completion operation if this code path is not taken.
|
||||
if constexpr (!__nothrow_decay_copyable<_As...>)
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(__state_->__rcvr_), execution::current_exception());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class... _As>
|
||||
_CCCL_HOST_DEVICE_API void set_value(_As&&... __as) noexcept
|
||||
{
|
||||
__set_result(set_value_t{}, static_cast<_As&&>(__as)...);
|
||||
execution::start(__state_->__opstate2_);
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API void set_error(_Error&& __error) noexcept
|
||||
{
|
||||
__set_result(set_error_t{}, static_cast<_Error&&>(__error));
|
||||
execution::start(__state_->__opstate2_);
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void set_stopped() noexcept
|
||||
{
|
||||
__set_result(set_stopped_t{});
|
||||
execution::start(__state_->__opstate2_);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __fwd_env_t<env_of_t<_Rcvr>>
|
||||
{
|
||||
return __fwd_env(execution::get_env(__state_->__rcvr_));
|
||||
}
|
||||
|
||||
__state_t<_Sch, _Rcvr, _Results>* __state_;
|
||||
};
|
||||
|
||||
template <class _Sch, class _CvSndr, class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
using __completions_t _CCCL_NODEBUG_ALIAS = completion_signatures_of_t<_CvSndr, __fwd_env_t<env_of_t<_Rcvr>>>;
|
||||
using __results_t _CCCL_NODEBUG_ALIAS =
|
||||
typename __completions_t::template __transform_q<::cuda::std::__decayed_tuple, __variant>;
|
||||
using __rcvr_t = continues_on_t::__rcvr_t<_Rcvr, __results_t>;
|
||||
using __stash_rcvr_t = continues_on_t::__stash_rcvr_t<_Sch, _Rcvr, __results_t>;
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __opstate_t(_CvSndr&& __sndr, _Sch __sch, _Rcvr __rcvr)
|
||||
: __state_{{static_cast<_Rcvr&&>(__rcvr), {}}, execution::connect(schedule(__sch), __rcvr_t{&__state_})}
|
||||
, __opstate1_{execution::connect(static_cast<_CvSndr&&>(__sndr), __stash_rcvr_t{&__state_})}
|
||||
{}
|
||||
|
||||
_CCCL_IMMOVABLE(__opstate_t);
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void start() noexcept
|
||||
{
|
||||
execution::start(__opstate1_);
|
||||
}
|
||||
|
||||
__state_t<_Sch, _Rcvr, __results_t> __state_;
|
||||
connect_result_t<_CvSndr, __stash_rcvr_t> __opstate1_;
|
||||
};
|
||||
|
||||
public:
|
||||
template <class _Sch, class _Sndr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __attrs_t;
|
||||
|
||||
template <class _Sch, class _Sndr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
template <class _Sch>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sndr>
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr __sndr) const
|
||||
-> __sndr_t<_Sch, __call_result_t<schedule_from_t, _Sndr>>
|
||||
{
|
||||
static_assert(__is_sender<_Sndr>);
|
||||
using __child_t = __call_result_t<schedule_from_t, _Sndr>;
|
||||
return __sndr_t<_Sch, __child_t>{{}, __sch_, schedule_from(static_cast<_Sndr&&>(__sndr))};
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sndr>
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API constexpr friend auto operator|(_Sndr __sndr, __closure_t __clsur)
|
||||
-> __sndr_t<_Sch, __call_result_t<schedule_from_t, _Sndr>>
|
||||
{
|
||||
static_assert(__is_sender<_Sndr>);
|
||||
using __child_t = __call_result_t<schedule_from_t, _Sndr>;
|
||||
return __sndr_t<_Sch, __child_t>{{}, __clsur.__sch_, schedule_from(static_cast<_Sndr&&>(__sndr))};
|
||||
}
|
||||
|
||||
_Sch __sch_;
|
||||
};
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sch>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sch __sch) const -> __closure_t<_Sch>
|
||||
{
|
||||
static_assert(__is_scheduler<_Sch>);
|
||||
return __closure_t<_Sch>{__sch};
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sch, class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr __sndr, _Sch __sch) const
|
||||
-> __sndr_t<_Sch, __call_result_t<schedule_from_t, _Sndr>>
|
||||
{
|
||||
static_assert(__is_sender<_Sndr>);
|
||||
static_assert(__is_scheduler<_Sch>);
|
||||
using __child_t = __call_result_t<schedule_from_t, _Sndr>;
|
||||
return __sndr_t<_Sch, __child_t>{{}, __sch, schedule_from(static_cast<_Sndr&&>(__sndr))};
|
||||
}
|
||||
};
|
||||
|
||||
//! @brief The @c continues_on sender's attributes.
|
||||
template <class _Sch, class _Sndr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT continues_on_t::__attrs_t
|
||||
{
|
||||
private:
|
||||
//! @brief Returns `true` when:
|
||||
//! - _SetTag is set_error_t, and
|
||||
//! - _Sndr has value completions, and
|
||||
//! - at least one of the value completions is not nothrow decay-copyable.
|
||||
//! In that case, error completions can come from the sender's value completions.
|
||||
template <class _SetTag, class... _Env>
|
||||
_CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL bool __has_decay_copy_errors() noexcept
|
||||
{
|
||||
if constexpr (__same_as<_SetTag, set_error_t>)
|
||||
{
|
||||
if constexpr (__has_completions_for<_Sndr, set_value_t, __fwd_env_t<_Env>...>)
|
||||
{
|
||||
using __completion_parts_t =
|
||||
__partitioned_completions_of_t<completion_signatures_of_t<_Sndr, __fwd_env_t<_Env>...>>;
|
||||
return !__completion_parts_t::__nothrow_decay_copyable::__values::value;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
const __sndr_t<_Sch, _Sndr>& __self_;
|
||||
|
||||
public:
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __attrs_t(const __sndr_t<_Sch, _Sndr>& __self) noexcept
|
||||
: __self_(__self)
|
||||
{}
|
||||
|
||||
//! @brief Queries the completion scheduler for a given @c _SetTag.
|
||||
//! @tparam _SetTag The completion tag to query for.
|
||||
//! @tparam _Env The environment to consider when querying for the completion
|
||||
//! scheduler.
|
||||
//!
|
||||
//! @note If @c _SetTag is @c set_value_t, then we are in the happy path: everything
|
||||
//! succeeded and execution continues on @c _Sch.
|
||||
//!
|
||||
//! Otherwise, if @c _Sndr never completes with @c _SetTag, and either @c _SetTag is
|
||||
//! @c set_stopped_t or decay-copying @c _Sndr's value results cannot throw, then a
|
||||
//! @c _SetTag completion can only come from the scheduler's sender. In this case, return
|
||||
//! the scheduler's completion scheduler if it has one.
|
||||
//!
|
||||
//! Otherwise, if the scheduler's sender never completes with @c _SetTag, then a
|
||||
//! @c _SetTag completion can only come from the original sender, so return the
|
||||
//! original sender's completion scheduler.
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _SetTag, class... _Env)
|
||||
_CCCL_REQUIRES((__same_as<_SetTag, set_value_t> || __never_completes_with<_Sndr, _SetTag, __fwd_env_t<_Env>...>)
|
||||
_CCCL_AND(!__has_decay_copy_errors<_SetTag, _Env...>()))
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_scheduler_t<_SetTag>, const _Env&... __env) const noexcept
|
||||
-> __call_result_t<get_completion_scheduler_t<_SetTag>, _Sch, __fwd_env_t<_Env>...>
|
||||
{
|
||||
return get_completion_scheduler<_SetTag>(__self_.__sch_, __fwd_env(__env)...);
|
||||
}
|
||||
|
||||
//! @overload
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _SetTag, class... _Env)
|
||||
_CCCL_REQUIRES(__never_completes_with<schedule_result_t<_Sch>, _SetTag, __fwd_env_t<_Env>...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_scheduler_t<_SetTag>, const _Env&... __env) const noexcept
|
||||
-> __call_result_t<get_completion_scheduler_t<_SetTag>, env_of_t<_Sndr>, __fwd_env_t<_Env>...>
|
||||
{
|
||||
return get_completion_scheduler<_SetTag>(get_env(__self_.__sndr_), __fwd_env(__env)...);
|
||||
}
|
||||
|
||||
//! @brief Queries the completion domain for a given @c _SetTag.
|
||||
//! @tparam _SetTag The completion tag to query for.
|
||||
//! @tparam _Env The environment to consider when querying for the completion domain.
|
||||
//!
|
||||
//! @note If @c _SetTag is @c set_value_t, then we are in the happy path: everything
|
||||
//! succeeded and execution continues on @c _Sch.
|
||||
//!
|
||||
//! Otherwise, if @c _SetTag is @c set_stopped_t or if decay-copying @c _Sndr's value
|
||||
//! results cannot throw, then a @c _SetTag completion can happen on the sender's
|
||||
//! completion domain (if it has one) or the scheduler's completion domain (if it has
|
||||
//! one).
|
||||
//!
|
||||
//! @note Otherwise, @c _SetTag is @c set_error_t and decay-copying @c _Sndr's value
|
||||
//! results can throw, so error completions can also come from the sender's value
|
||||
//! completions.
|
||||
_CCCL_TEMPLATE(class _SetTag, class... _Env)
|
||||
_CCCL_REQUIRES(__same_as<_SetTag, set_value_t>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_domain_t<_SetTag>, const _Env&...) const noexcept
|
||||
-> __unless_one_of_t<__compl_domain_t<_SetTag, schedule_result_t<_Sch>, __fwd_env_t<_Env>...>, __not_a_domain>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
//! @overload
|
||||
_CCCL_TEMPLATE(class _SetTag, class... _Env)
|
||||
_CCCL_REQUIRES((!__same_as<_SetTag, set_value_t>) _CCCL_AND(!__has_decay_copy_errors<_SetTag, _Env...>()))
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_domain_t<_SetTag>, const _Env&...) const noexcept
|
||||
-> __unless_one_of_t<__common_domain_t<__compl_domain_t<_SetTag, _Sndr, __fwd_env_t<_Env>...>,
|
||||
__compl_domain_t<_SetTag, schedule_result_t<_Sch>, __fwd_env_t<_Env>...>>,
|
||||
__not_a_domain>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
//! @overload
|
||||
_CCCL_TEMPLATE(class _SetTag, class... _Env)
|
||||
_CCCL_REQUIRES((__has_decay_copy_errors<_SetTag, _Env...>()))
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_domain_t<_SetTag>, const _Env&...) const noexcept
|
||||
-> __unless_one_of_t<__common_domain_t<__compl_domain_t<_SetTag, _Sndr, __fwd_env_t<_Env>...>,
|
||||
__compl_domain_t<_SetTag, schedule_result_t<_Sch>, __fwd_env_t<_Env>...>,
|
||||
__compl_domain_t<set_value_t, _Sndr, __fwd_env_t<_Env>...>>,
|
||||
__not_a_domain>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
//! @brief Queries the completion behavior of the combined sender.
|
||||
//! @tparam _Env The environment to consider when querying for the completion behavior.
|
||||
//! @note The completion behavior is the minimum between the scheduler's sender and
|
||||
//! the original sender.
|
||||
template <class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_behavior_t, const _Env&...) const noexcept
|
||||
{
|
||||
return (execution::min) (execution::get_completion_behavior<schedule_result_t<_Sch>, __fwd_env_t<_Env>...>(),
|
||||
execution::get_completion_behavior<_Sndr, _Env...>());
|
||||
}
|
||||
|
||||
//! @brief Forwards other queries to the underlying sender's environment.
|
||||
//! @pre @c _Tag is a forwarding query but not a completion query.
|
||||
_CCCL_TEMPLATE(class _Tag, class... _Args)
|
||||
_CCCL_REQUIRES(__forwarding_query<_Tag> _CCCL_AND(!__is_completion_query<_Tag>)
|
||||
_CCCL_AND __queryable_with<env_of_t<_Sndr>, _Tag, _Args...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(_Tag, _Args&&... __args) const
|
||||
noexcept(__nothrow_queryable_with<env_of_t<_Sndr>, _Tag, _Args...>)
|
||||
-> __query_result_t<env_of_t<_Sndr>, _Tag, _Args...>
|
||||
{
|
||||
return get_env(__self_.__sndr_).query(_Tag{}, static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// continues_on sender
|
||||
template <class _Sch, class _Sndr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT continues_on_t::__sndr_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class _Self, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
_CUDAX_LET_COMPLETIONS(auto(__child_completions) = get_child_completion_signatures<_Self, _Sndr, _Env...>())
|
||||
{
|
||||
_CUDAX_LET_COMPLETIONS(
|
||||
auto(__sch_completions) = execution::get_completion_signatures<schedule_result_t<_Sch>, __fwd_env_t<_Env>...>())
|
||||
{
|
||||
// The scheduler contributes error and stopped completions.
|
||||
return concat_completion_signatures(
|
||||
transform_completion_signatures(__sch_completions, __swallow_transform{}),
|
||||
transform_completion_signatures(
|
||||
__child_completions, __detail::__decay_args<set_value_t>{}, __detail::__decay_args<set_error_t>{}));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) && -> __opstate_t<_Sch, _Sndr, _Rcvr>
|
||||
{
|
||||
return __opstate_t<_Sch, _Sndr, _Rcvr>{static_cast<_Sndr&&>(__sndr_), __sch_, static_cast<_Rcvr&&>(__rcvr)};
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
connect(_Rcvr __rcvr) const& -> __opstate_t<_Sch, const _Sndr&, _Rcvr>
|
||||
{
|
||||
return __opstate_t<_Sch, const _Sndr&, _Rcvr>{__sndr_, __sch_, static_cast<_Rcvr&&>(__rcvr)};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __attrs_t<_Sch, _Sndr>
|
||||
{
|
||||
return __attrs_t<_Sch, _Sndr>(*this);
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ continues_on_t __tag_;
|
||||
_Sch __sch_;
|
||||
_Sndr __sndr_;
|
||||
};
|
||||
|
||||
template <class _Sch, class _Sndr>
|
||||
inline constexpr int structured_binding_size<continues_on_t::__sndr_t<_Sch, _Sndr>> = 3;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT continues_on_t continues_on{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_CONTINUES_ON
|
||||
@@ -1,191 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_CPOS
|
||||
#define __CUDAX_EXECUTION_CPOS
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__type_traits/is_same.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/transform_sender.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
// make the completion tags equality comparable
|
||||
template <__disposition _Disposition>
|
||||
struct __completion_tag
|
||||
{
|
||||
template <__disposition _OtherDisposition>
|
||||
_CCCL_TRIVIAL_API constexpr auto operator==(__completion_tag<_OtherDisposition>) const noexcept -> bool
|
||||
{
|
||||
return _Disposition == _OtherDisposition;
|
||||
}
|
||||
|
||||
template <__disposition _OtherDisposition>
|
||||
_CCCL_TRIVIAL_API constexpr auto operator!=(__completion_tag<_OtherDisposition>) const noexcept -> bool
|
||||
{
|
||||
return _Disposition != _OtherDisposition;
|
||||
}
|
||||
|
||||
static constexpr __disposition __disposition = _Disposition;
|
||||
};
|
||||
|
||||
template <class _Rcvr, class... _Ts>
|
||||
_CCCL_CONCEPT __has_set_value_mbr = //
|
||||
_CCCL_REQUIRES_EXPR((_Rcvr, variadic _Ts), _Rcvr& __rcvr) //
|
||||
( //
|
||||
static_cast<_Rcvr&&>(__rcvr).set_value(::cuda::std::declval<_Ts>()...) //
|
||||
);
|
||||
|
||||
struct set_value_t : __completion_tag<__disposition::__value>
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Rcvr, class... _Ts)
|
||||
_CCCL_REQUIRES(__has_set_value_mbr<_Rcvr, _Ts...>)
|
||||
_CCCL_TRIVIAL_API constexpr void operator()(_Rcvr&& __rcvr, _Ts&&... __ts) const noexcept
|
||||
{
|
||||
static_assert(__same_as<decltype(static_cast<_Rcvr&&>(__rcvr).set_value(static_cast<_Ts&&>(__ts)...)), void>);
|
||||
static_assert(noexcept(static_cast<_Rcvr&&>(__rcvr).set_value(static_cast<_Ts&&>(__ts)...)));
|
||||
static_cast<_Rcvr&&>(__rcvr).set_value(static_cast<_Ts&&>(__ts)...);
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Rcvr, class _Ey>
|
||||
_CCCL_CONCEPT __has_set_error_mbr = //
|
||||
_CCCL_REQUIRES_EXPR((_Rcvr, _Ey), _Rcvr& __rcvr, _Ey&& __e) //
|
||||
( //
|
||||
static_cast<_Rcvr&&>(__rcvr).set_error(static_cast<_Ey&&>(__e)) //
|
||||
);
|
||||
|
||||
struct set_error_t : __completion_tag<__disposition::__error>
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Rcvr, class _Ey)
|
||||
_CCCL_REQUIRES(__has_set_error_mbr<_Rcvr, _Ey>)
|
||||
_CCCL_TRIVIAL_API constexpr void operator()(_Rcvr&& __rcvr, _Ey&& __e) const noexcept
|
||||
{
|
||||
static_assert(__same_as<decltype(static_cast<_Rcvr&&>(__rcvr).set_error(static_cast<_Ey&&>(__e))), void>);
|
||||
static_assert(noexcept(static_cast<_Rcvr&&>(__rcvr).set_error(static_cast<_Ey&&>(__e))));
|
||||
static_cast<_Rcvr&&>(__rcvr).set_error(static_cast<_Ey&&>(__e));
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Rcvr>
|
||||
_CCCL_CONCEPT __has_set_stopped_mbr = //
|
||||
_CCCL_REQUIRES_EXPR((_Rcvr), _Rcvr& __rcvr) //
|
||||
( //
|
||||
static_cast<_Rcvr&&>(__rcvr).set_stopped() //
|
||||
);
|
||||
|
||||
struct set_stopped_t : __completion_tag<__disposition::__stopped>
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Rcvr)
|
||||
_CCCL_REQUIRES(__has_set_stopped_mbr<_Rcvr>)
|
||||
_CCCL_TRIVIAL_API constexpr void operator()(_Rcvr&& __rcvr) const noexcept
|
||||
{
|
||||
static_assert(__same_as<decltype(static_cast<_Rcvr&&>(__rcvr).set_stopped()), void>);
|
||||
static_assert(noexcept(static_cast<_Rcvr&&>(__rcvr).set_stopped()));
|
||||
static_cast<_Rcvr&&>(__rcvr).set_stopped();
|
||||
}
|
||||
};
|
||||
|
||||
template <class _OpState>
|
||||
_CCCL_CONCEPT __has_start_mbr = //
|
||||
_CCCL_REQUIRES_EXPR((_OpState), _OpState& __opstate) //
|
||||
( //
|
||||
__opstate.start() //
|
||||
);
|
||||
|
||||
struct start_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _OpState)
|
||||
_CCCL_REQUIRES(__has_start_mbr<_OpState>)
|
||||
_CCCL_TRIVIAL_API constexpr void operator()(_OpState& __opstate) const noexcept
|
||||
{
|
||||
static_assert(__same_as<decltype(__opstate.start()), void>);
|
||||
static_assert(noexcept(__opstate.start()));
|
||||
__opstate.start();
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Sndr, class _Rcvr>
|
||||
_CCCL_CONCEPT __has_connect_mbr = //
|
||||
_CCCL_REQUIRES_EXPR((_Sndr, _Rcvr), _Sndr& __sndr, _Rcvr& __rcvr) //
|
||||
( //
|
||||
static_cast<_Sndr&&>(__sndr).connect(static_cast<_Rcvr&&>(__rcvr)) //
|
||||
);
|
||||
|
||||
// connect
|
||||
struct connect_t
|
||||
{
|
||||
private:
|
||||
template <class _Sndr, class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto __get_declfn() noexcept
|
||||
{
|
||||
using __new_sender_t = transform_sender_result_t<_Sndr, env_of_t<_Rcvr>>;
|
||||
if constexpr (__has_connect_mbr<__new_sender_t, _Rcvr>)
|
||||
{
|
||||
constexpr auto __sndr = __declfn<_Sndr>;
|
||||
constexpr auto __rcvr = __declfn<_Rcvr>;
|
||||
using __result_t = decltype(transform_sender(__sndr(), get_env(__rcvr())).connect(__rcvr()));
|
||||
constexpr bool __is_nothrow = noexcept(transform_sender(__sndr(), get_env(__rcvr())).connect(__rcvr()));
|
||||
return __declfn<__result_t, __is_nothrow>;
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
template <class _Sndr, class _Rcvr, auto _DeclFn = __get_declfn<_Sndr, _Rcvr>()>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr&& __sndr, _Rcvr __rcvr) const
|
||||
noexcept(noexcept(_DeclFn())) -> decltype(_DeclFn())
|
||||
{
|
||||
auto&& __env = get_env(__rcvr);
|
||||
return transform_sender(static_cast<_Sndr&&>(__sndr), static_cast<decltype(__env)>(__env))
|
||||
.connect(static_cast<_Rcvr&&>(__rcvr));
|
||||
}
|
||||
};
|
||||
|
||||
struct schedule_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sch>
|
||||
_CCCL_TRIVIAL_API constexpr auto operator()(_Sch&& __sch) const noexcept
|
||||
{
|
||||
static_assert(noexcept(static_cast<_Sch&&>(__sch).schedule()));
|
||||
return static_cast<_Sch&&>(__sch).schedule();
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT set_value_t set_value{};
|
||||
_CCCL_GLOBAL_CONSTANT set_error_t set_error{};
|
||||
_CCCL_GLOBAL_CONSTANT set_stopped_t set_stopped{};
|
||||
_CCCL_GLOBAL_CONSTANT start_t start{};
|
||||
_CCCL_GLOBAL_CONSTANT connect_t connect{};
|
||||
_CCCL_GLOBAL_CONSTANT schedule_t schedule{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_CPOS
|
||||
@@ -1,173 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_DIAGNOSTICS
|
||||
#define __CUDAX_EXECUTION_DIAGNOSTICS
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
// The following must be left undefined
|
||||
template <class...>
|
||||
struct _DIAGNOSTIC;
|
||||
|
||||
struct _UNKNOWN;
|
||||
|
||||
struct _WHERE;
|
||||
|
||||
struct _WHAT;
|
||||
|
||||
struct _TO_FIX_THIS_ERROR;
|
||||
|
||||
struct _IN_ALGORITHM;
|
||||
|
||||
struct _WITH_FUNCTION;
|
||||
|
||||
struct _WITH_SENDER;
|
||||
|
||||
struct _WITH_ARGUMENTS;
|
||||
|
||||
struct _WITH_QUERY;
|
||||
|
||||
struct _WITH_ENVIRONMENT;
|
||||
|
||||
struct _WITH_SIGNATURES;
|
||||
|
||||
template <class>
|
||||
struct _WITH_COMPLETION_SIGNATURE;
|
||||
|
||||
struct _FUNCTION_IS_NOT_CALLABLE;
|
||||
|
||||
struct _FUNCTION_MUST_RETURN_A_SENDER;
|
||||
|
||||
struct _FUNCTION_MUST_RETURN_SENDERS_THAT_ALL_COMPLETE_IN_A_COMMON_DOMAIN;
|
||||
|
||||
struct _WITH_RETURN_TYPE;
|
||||
|
||||
struct _SENDER_HAS_TOO_MANY_SUCCESS_COMPLETIONS;
|
||||
|
||||
struct _ARGUMENTS_ARE_NOT_DECAY_COPYABLE;
|
||||
|
||||
struct _THE_ENVIRONMENT_OF_THE_RECEIVER_DOES_NOT_HAVE_A_SCHEDULER_FOR_ON_TO_RETURN_TO;
|
||||
|
||||
struct __merror_base
|
||||
{
|
||||
// _CCCL_HIDE_FROM_ABI virtual ~__merror_base() = default;
|
||||
|
||||
_CCCL_HOST_DEVICE friend constexpr auto __ustdex_unhandled_error(void*) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
template <class... _What>
|
||||
struct _ERROR : __merror_base
|
||||
{
|
||||
// The following aliases are to simplify error propagation
|
||||
// in the completion signatures meta-programming.
|
||||
template <class...>
|
||||
using __call _CCCL_NODEBUG_ALIAS = _ERROR;
|
||||
|
||||
using __partitioned _CCCL_NODEBUG_ALIAS = _ERROR;
|
||||
|
||||
template <template <class...> class, template <class...> class>
|
||||
using __value_types _CCCL_NODEBUG_ALIAS = _ERROR;
|
||||
|
||||
template <template <class...> class>
|
||||
using __error_types _CCCL_NODEBUG_ALIAS = _ERROR;
|
||||
|
||||
using __sends_stopped _CCCL_NODEBUG_ALIAS = _ERROR;
|
||||
|
||||
// The following operator overloads also simplify error propagation.
|
||||
_CCCL_HOST_DEVICE auto operator+() -> _ERROR;
|
||||
|
||||
template <class _Ty>
|
||||
_CCCL_HOST_DEVICE auto operator,(_Ty&) -> _ERROR&;
|
||||
|
||||
template <class... _With>
|
||||
_CCCL_HOST_DEVICE auto with(_ERROR<_With...>&) -> _ERROR<_What..., _With...>&;
|
||||
};
|
||||
|
||||
_CCCL_HOST_DEVICE constexpr auto __ustdex_unhandled_error(...) noexcept -> bool
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
template <class _Ty>
|
||||
inline constexpr bool __type_is_error = false;
|
||||
|
||||
template <class... _What>
|
||||
inline constexpr bool __type_is_error<_ERROR<_What...>> = true;
|
||||
|
||||
template <class... _What>
|
||||
inline constexpr bool __type_is_error<_ERROR<_What...>&> = true;
|
||||
|
||||
// True if any of the types in _Ts... are errors; false otherwise.
|
||||
template <class... _Ts>
|
||||
inline constexpr bool __type_contains_error =
|
||||
#if _CCCL_COMPILER(MSVC)
|
||||
(__type_is_error<_Ts> || ...);
|
||||
#else
|
||||
__ustdex_unhandled_error(static_cast<::cuda::std::__type_list<_Ts...>*>(nullptr));
|
||||
#endif
|
||||
|
||||
template <class... _Ts>
|
||||
using __type_find_error _CCCL_NODEBUG_ALIAS = decltype(+(declval<_Ts&>(), ..., declval<_ERROR<_UNKNOWN>&>()));
|
||||
|
||||
template <class... _What>
|
||||
struct __not_a_sender
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class...>
|
||||
_CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
return execution::invalid_completion_signature<_What...>();
|
||||
}
|
||||
};
|
||||
|
||||
template <class... _What>
|
||||
struct __not_a_scheduler
|
||||
{
|
||||
using scheduler_concept = scheduler_t;
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto schedule() noexcept
|
||||
{
|
||||
return __not_a_sender<_What...>{};
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr bool operator==(__not_a_scheduler) const noexcept
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr bool operator!=(__not_a_scheduler) const noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_DIAGNOSTICS
|
||||
@@ -1,486 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_DOMAIN
|
||||
#define __CUDAX_EXECUTION_DOMAIN
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__concepts/same_as.h>
|
||||
#include <cuda/std/__execution/env.h>
|
||||
#include <cuda/std/__tuple_dir/ignore.h>
|
||||
#include <cuda/std/__type_traits/common_type.h>
|
||||
#include <cuda/std/__type_traits/conditional.h>
|
||||
#include <cuda/std/__type_traits/decay.h>
|
||||
#include <cuda/std/__type_traits/is_empty.h>
|
||||
#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
|
||||
#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
|
||||
#include <cuda/std/__utility/undefined.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/completion_behavior.cuh>
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
template <class _DomainOrTag, class... _Args>
|
||||
using __apply_sender_result_t _CCCL_NODEBUG_ALIAS = decltype(_DomainOrTag{}.apply_sender(declval<_Args>()...));
|
||||
|
||||
// _DomainOrTag: eg, default_domain or then_t
|
||||
// _OpTag: either start_t or set_value_t
|
||||
template <class _DomainOrTag, class _OpTag, class _Sndr, class... _Env>
|
||||
using __transform_sender_result_t =
|
||||
decltype(declval<_DomainOrTag>().transform_sender(declval<_OpTag>(), declval<_Sndr>(), declval<const _Env&>()...));
|
||||
|
||||
template <class _DomainOrTag, class _OpTag, class _Sndr, class... _Env>
|
||||
_CCCL_CONCEPT __has_transform_sender =
|
||||
__is_instantiable_with<__transform_sender_result_t, _DomainOrTag, _OpTag, _Sndr, _Env...>;
|
||||
|
||||
template <class _DomainOrTag, class _OpTag, class _Sndr, class... _Env>
|
||||
_CCCL_CONCEPT __nothrow_transform_sender =
|
||||
_CCCL_REQUIRES_EXPR((_DomainOrTag, _OpTag, _Sndr, variadic _Env), __declfn_t<_Sndr> __sndr, const _Env&... __env) //
|
||||
( //
|
||||
noexcept(_DomainOrTag{}.transform_sender(_OpTag{}, __sndr(), __env...)) //
|
||||
);
|
||||
|
||||
template <class _Domain>
|
||||
_CCCL_CONCEPT __domain_like =
|
||||
::cuda::std::is_empty_v<_Domain> && //
|
||||
::cuda::std::is_nothrow_default_constructible_v<_Domain> && //
|
||||
::cuda::std::is_nothrow_copy_constructible_v<_Domain>;
|
||||
|
||||
//! @brief A structure that selects the default set of algorithm implementations for
|
||||
//! senders.
|
||||
//!
|
||||
//! This structure defines static member functions to handle operations on senders, such
|
||||
//! as applying and transforming them. It is designed to work with CUDA's experimental
|
||||
//! execution framework.
|
||||
//! @see https://eel.is/c++draft/exec.domain.default
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT default_domain
|
||||
{
|
||||
//! @brief Applies a sender operation using the specified tag and arguments.
|
||||
//!
|
||||
//! @tparam _Tag The tag type that defines the operation to be applied.
|
||||
//! @tparam _Sndr The type of the sender.
|
||||
//! @tparam _Args Variadic template for additional arguments.
|
||||
//! @param _Tag The tag instance specifying the operation.
|
||||
//! @param __sndr The sender to which the operation is applied.
|
||||
//! @param __args Additional arguments for the operation.
|
||||
//! @return The result of applying the sender operation.
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Tag, class _Sndr, class... _Args>
|
||||
_CCCL_HOST_DEVICE_API static constexpr auto apply_sender(_Tag, _Sndr&& __sndr, _Args&&... __args) noexcept(
|
||||
noexcept(_Tag{}.apply_sender(declval<_Sndr>(), declval<_Args>()...))) //
|
||||
-> __apply_sender_result_t<_Tag, _Sndr, _Args...>
|
||||
{
|
||||
return _Tag{}.apply_sender(static_cast<_Sndr&&>(__sndr), static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
//! @brief Transforms a sender with an environment.
|
||||
//!
|
||||
//! @tparam _OpTag Either start_t or set_value_t.
|
||||
//! @tparam _Sndr The type of the sender.
|
||||
//! @tparam _Env The type of the environment.
|
||||
//! @param __sndr The sender to be transformed.
|
||||
//! @param __env The environment used for the transformation.
|
||||
//! @return The result of transforming the sender with the given environment.
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _OpTag, class _Sndr, class _Env)
|
||||
_CCCL_REQUIRES(__has_transform_sender<tag_of_t<_Sndr>, _OpTag, _Sndr, _Env>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto
|
||||
transform_sender(_OpTag, _Sndr&& __sndr, const _Env& __env) //
|
||||
noexcept(__nothrow_transform_sender<tag_of_t<_Sndr>, _OpTag, _Sndr, _Env>)
|
||||
-> __transform_sender_result_t<tag_of_t<_Sndr>, _OpTag, _Sndr, _Env>
|
||||
{
|
||||
return tag_of_t<_Sndr>{}.transform_sender(_OpTag{}, static_cast<_Sndr&&>(__sndr), __env);
|
||||
}
|
||||
|
||||
//! @overload
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto
|
||||
transform_sender(::cuda::std::__ignore_t, _Sndr&& __sndr, ::cuda::std::__ignore_t) //
|
||||
noexcept(__nothrow_movable<_Sndr>) -> _Sndr
|
||||
{
|
||||
return static_cast<_Sndr&&>(__sndr);
|
||||
}
|
||||
};
|
||||
|
||||
//! @brief Concept that checks whether a domain's sender transform behaves like that of
|
||||
//! @c default_domain when passed the same arguments. The concept is modeled when either
|
||||
//! of the following is
|
||||
template <class _Domain, class _OpTag, class _Sndr, class _Env>
|
||||
_CCCL_CONCEPT __default_domain_like =
|
||||
__same_as<decay_t<__transform_sender_result_t<default_domain, _OpTag, _Sndr, _Env>>,
|
||||
decay_t<::cuda::std::__type_call<
|
||||
::cuda::std::__type_try_catch<
|
||||
::cuda::std::__type_quote<__transform_sender_result_t>,
|
||||
::cuda::std::__type_always<__transform_sender_result_t<default_domain, _OpTag, _Sndr, _Env>>>,
|
||||
_Domain,
|
||||
_OpTag,
|
||||
_Sndr,
|
||||
_Env>>>;
|
||||
|
||||
/**
|
||||
* @brief Tag type representing an indeterminate (unspecified) execution domain.
|
||||
*
|
||||
* This domain tag is used when a sender can complete with a given disposition
|
||||
* from multiple execution domains.
|
||||
*
|
||||
* @tparam _Domains...: the (possibly empty) set of domains that a sender's
|
||||
* completion may originate from.
|
||||
*/
|
||||
template <class... _Domains>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT indeterminate_domain
|
||||
{
|
||||
_CCCL_HIDE_FROM_ABI indeterminate_domain() = default;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr indeterminate_domain(::cuda::std::__ignore_t) noexcept {}
|
||||
|
||||
//! @brief Transforms a sender with an optional environment.
|
||||
//!
|
||||
//! @tparam _OpTag Either start_t or set_value_t.
|
||||
//! @tparam _Sndr The type of the sender.
|
||||
//! @tparam _Env The type of the environment.
|
||||
//! @param __sndr The sender to be transformed.
|
||||
//! @param __env The environment used for the transformation.
|
||||
//! @return `default_domain{}.transform_sender(_OpTag{}, std::forward<_Sndr>(__sndr), __env)`
|
||||
//! @pre Every type in @c _Domains... must behave like @c default_domain when passed the
|
||||
//! same arguments. If this check fails, the @c static_assert triggers with: "ERROR:
|
||||
//! indeterminate domains: cannot pick an algorithm customization"
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _OpTag, class _Sndr, class _Env)
|
||||
_CCCL_REQUIRES(__has_transform_sender<tag_of_t<_Sndr>, _OpTag, _Sndr, _Env>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto
|
||||
transform_sender(_OpTag, _Sndr&& __sndr, const _Env& __env) //
|
||||
noexcept(__nothrow_transform_sender<tag_of_t<_Sndr>, _OpTag, _Sndr, _Env>)
|
||||
-> __transform_sender_result_t<tag_of_t<_Sndr>, _OpTag, _Sndr, _Env>
|
||||
{
|
||||
static_assert((__default_domain_like<_Domains, _OpTag, _Sndr, _Env> && ...),
|
||||
"ERROR: indeterminate domains: cannot pick an algorithm customization");
|
||||
return tag_of_t<_Sndr>{}.transform_sender(_OpTag{}, static_cast<_Sndr&&>(__sndr), __env);
|
||||
}
|
||||
};
|
||||
|
||||
//! @brief A wrapper around an environment that hides a set of queries.
|
||||
template <class _Env, class... _Queries>
|
||||
struct __hide_query
|
||||
{
|
||||
static_assert(__nothrow_movable<_Env>);
|
||||
|
||||
_CCCL_HOST_DEVICE_API explicit constexpr __hide_query(_Env&& __env, _Queries...) noexcept
|
||||
: __env_{static_cast<_Env&&>(__env)}
|
||||
{}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Query, class... _As)
|
||||
_CCCL_REQUIRES(__none_of<_Query, _Queries...> _CCCL_AND __queryable_with<_Env, _Query, _As...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Query __query, const _As&... __as) const
|
||||
noexcept(__nothrow_queryable_with<_Env, _Query, _As...>) -> __query_result_t<_Env, _Query, _As...>
|
||||
{
|
||||
return __env_.query(__query, __as...);
|
||||
}
|
||||
|
||||
private:
|
||||
_Env __env_;
|
||||
};
|
||||
|
||||
template <class _Env>
|
||||
struct __hide_scheduler : __hide_query<_Env, get_scheduler_t, get_domain_t>
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API explicit constexpr __hide_scheduler(_Env&& __env) noexcept
|
||||
: __hide_query<_Env, get_scheduler_t, get_domain_t>{static_cast<_Env&&>(__env), {}, {}}
|
||||
{}
|
||||
};
|
||||
|
||||
template <class _Env>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES __hide_scheduler(_Env&&) -> __hide_scheduler<_Env>;
|
||||
|
||||
template <class _Sch, class... _Env>
|
||||
using __scheduler_domain_t _CCCL_NODEBUG_ALIAS = __call_result_t<get_completion_domain_t<set_value_t>, _Sch, _Env...>;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
//! @brief A query type for asking a receiver's environment for its domain, which is an
|
||||
//! empty class type that is used in tag dispatching to find a custom implementation of a
|
||||
//! sender algorithm. The result of this query is the "current" domain; that is, the domain
|
||||
//! where `start` will be called on the operation state that results from connecting the
|
||||
//! receiver to a sender.
|
||||
struct get_domain_t
|
||||
{
|
||||
//! @brief If there is a @c get_domain_t query in @c __env, return it.
|
||||
_CCCL_TEMPLATE(class _Env)
|
||||
_CCCL_REQUIRES(__queryable_with<_Env, get_domain_t>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Env&) const noexcept
|
||||
-> decay_t<__query_result_t<_Env, get_domain_t>>
|
||||
{
|
||||
using __domain_t = decay_t<__query_result_t<_Env, get_domain_t>>;
|
||||
static_assert(__domain_like<__domain_t>, "Domain types are required to be empty class types");
|
||||
return __domain_t{};
|
||||
}
|
||||
|
||||
//! @brief If there is not a @c get_domain_t query in @c __env, but there is a
|
||||
//! scheduler, return the domain of the scheduler if it has one, and @c default_domain
|
||||
//! otherwise.
|
||||
_CCCL_TEMPLATE(class _Env)
|
||||
_CCCL_REQUIRES((!__queryable_with<_Env, get_domain_t>) _CCCL_AND __callable<get_scheduler_t, const _Env&>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Env&) const noexcept
|
||||
{
|
||||
using __sch_t = __scheduler_of_t<const _Env&>;
|
||||
using __env_t = __hide_scheduler<const _Env&>; // to prevent recursion
|
||||
using __cmpl_sch_t = __call_result_or_t<get_completion_scheduler_t<set_value_t>, __sch_t, __sch_t, __env_t>;
|
||||
using __domain_t = __scheduler_domain_t<__cmpl_sch_t, __env_t>;
|
||||
static_assert(__domain_like<__domain_t>, "Domain types are required to be empty class types");
|
||||
return __domain_t{};
|
||||
}
|
||||
|
||||
//! @brief Fall back to the default domain if no other domain is found.
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::std::__ignore_t) const noexcept
|
||||
-> default_domain
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API static constexpr auto query(forwarding_query_t) noexcept
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT get_domain_t get_domain{};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
//! @brief A query type for asking a sender's attributes for the domain on which that
|
||||
//! sender will complete. As with @c get_domain, it is used in tag dispatching to find a
|
||||
//! custom implementation of a sender algorithm.
|
||||
//!
|
||||
//! @tparam _Tag one of set_value_t, set_error_t, or set_stopped_t
|
||||
template <class _Tag>
|
||||
struct get_completion_domain_t
|
||||
{
|
||||
// This function object reads the completion domain from an attribute object or a
|
||||
// scheduler, accounting for the fact that the query member function may or may not
|
||||
// accept an environment.
|
||||
struct __read_query_t
|
||||
{
|
||||
_CCCL_TEMPLATE(class _Attrs)
|
||||
_CCCL_REQUIRES(__queryable_with<_Attrs, get_completion_domain_t>)
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(const _Attrs&, cuda::std::__ignore_t = {}) const noexcept
|
||||
{
|
||||
return decay_t<__query_result_t<_Attrs, get_completion_domain_t>>{};
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Attrs, class _Env)
|
||||
_CCCL_REQUIRES(__queryable_with<_Attrs, get_completion_domain_t, const _Env&>)
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(const _Attrs&, const _Env&) const noexcept
|
||||
{
|
||||
return decay_t<__query_result_t<_Attrs, get_completion_domain_t, const _Env&>>{};
|
||||
}
|
||||
};
|
||||
|
||||
private:
|
||||
template <class _Sch, class Domain, class... _Env>
|
||||
_CCCL_HOST_DEVICE_API static constexpr void __check_scheduler_domain() noexcept
|
||||
{
|
||||
static_assert(__same_as<Domain, __scheduler_domain_t<_Sch, const _Env&...>>,
|
||||
"the sender's completion scheduler's domain does not match the domain returned by the scheduler");
|
||||
}
|
||||
|
||||
template <class _Attrs, class... _Env, class _Domain>
|
||||
[[nodiscard]] _CCCL_TRIVIAL_API static _CCCL_CONSTEVAL auto __check_domain(_Domain) noexcept
|
||||
{
|
||||
// Sanity check: if a completion scheduler can be determined, then its domain must match
|
||||
// the domain returned by the attributes.
|
||||
if constexpr (__callable<get_completion_scheduler_t<_Tag>, const _Attrs&, const _Env&...>)
|
||||
{
|
||||
using __sch_t = decay_t<__call_result_t<get_completion_scheduler_t<_Tag>, const _Attrs&, const _Env&...>>;
|
||||
if constexpr (!__same_as<__sch_t, _Attrs>) // prevent infinite recursion
|
||||
{
|
||||
get_completion_domain_t::__check_scheduler_domain<__sch_t, _Domain, _Env...>();
|
||||
}
|
||||
}
|
||||
return __declfn<_Domain>;
|
||||
}
|
||||
|
||||
template <class _Attrs, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto __get_declfn() noexcept
|
||||
{
|
||||
// If __attrs has a completion domain, then return it:
|
||||
if constexpr (__callable<__read_query_t, const _Attrs&, const _Env&...>)
|
||||
{
|
||||
using __domain_t = __call_result_t<__read_query_t, const _Attrs&, const _Env&...>;
|
||||
static_assert(__domain_like<__domain_t>, "Domain types are required to be empty class types");
|
||||
return __check_domain<_Attrs, _Env...>(__domain_t{});
|
||||
}
|
||||
// Otherwise, if __attrs has a completion scheduler, we can ask that scheduler for its
|
||||
// completion domain.
|
||||
else if constexpr (__callable<get_completion_scheduler_t<_Tag>, const _Attrs&, const _Env&...>)
|
||||
{
|
||||
using __sch_t = __call_result_t<get_completion_scheduler_t<_Tag>, const _Attrs&, const _Env&...>;
|
||||
using __read_query_t = typename get_completion_domain_t<set_value_t>::__read_query_t;
|
||||
|
||||
if constexpr (__callable<__read_query_t, __sch_t, const _Env&...>)
|
||||
{
|
||||
using __domain_t = __call_result_t<__read_query_t, __sch_t, const _Env&...>;
|
||||
static_assert(__domain_like<__domain_t>, "Domain types are required to be empty class types");
|
||||
return __declfn<__domain_t>;
|
||||
}
|
||||
// Otherwise, if the scheduler's sender indicates that it completes inline, we can ask
|
||||
// the environment for its domain.
|
||||
else if constexpr (__completes_inline<env_of_t<schedule_result_t<__sch_t>>, _Env...>
|
||||
&& __callable<get_domain_t, const _Env&...>)
|
||||
{
|
||||
using __domain_t = __call_result_t<get_domain_t, const _Env&...>;
|
||||
return __declfn<__domain_t>;
|
||||
}
|
||||
// Otherwise, if we are asking "late" (with an environment), return the default_domain
|
||||
else if constexpr (sizeof...(_Env) != 0)
|
||||
{
|
||||
return __declfn<default_domain>;
|
||||
}
|
||||
}
|
||||
// Otherwise, if the attributes indicates that the sender completes inline, we can ask
|
||||
// the environment for its domain.
|
||||
else if constexpr (__completes_inline<_Attrs, _Env...> && __callable<get_domain_t, const _Env&...>)
|
||||
{
|
||||
using __domain_t = __call_result_t<get_domain_t, const _Env&...>;
|
||||
return __declfn<__domain_t>;
|
||||
}
|
||||
// Otherwise, if we are asking "late" (with an environment), return the default_domain
|
||||
else if constexpr (sizeof...(_Env) != 0)
|
||||
{
|
||||
return __declfn<default_domain>;
|
||||
}
|
||||
// Otherwise, no completion domain can be determined. Return void.
|
||||
}
|
||||
|
||||
public:
|
||||
template <class _Attrs, class... _Env, auto _DeclFn = __get_declfn<_Attrs, _Env...>()>
|
||||
[[nodiscard]] _CCCL_TRIVIAL_API constexpr auto operator()(const _Attrs&, const _Env&...) const noexcept
|
||||
-> decltype(_DeclFn())
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_TRIVIAL_API static constexpr auto query(forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Tag>
|
||||
extern ::cuda::std::__undefined<_Tag> get_completion_domain;
|
||||
|
||||
// Explicitly instantiate these because of variable template weirdness in device code
|
||||
template <>
|
||||
_CCCL_GLOBAL_CONSTANT get_completion_domain_t<set_value_t> get_completion_domain<set_value_t>{};
|
||||
template <>
|
||||
_CCCL_GLOBAL_CONSTANT get_completion_domain_t<set_error_t> get_completion_domain<set_error_t>{};
|
||||
template <>
|
||||
_CCCL_GLOBAL_CONSTANT get_completion_domain_t<set_stopped_t> get_completion_domain<set_stopped_t>{};
|
||||
|
||||
struct __not_a_domain
|
||||
{
|
||||
_CCCL_HIDE_FROM_ABI __not_a_domain() = default;
|
||||
template <class _Domain>
|
||||
_CCCL_HOST_DEVICE_API constexpr __not_a_domain(_Domain&&) noexcept
|
||||
{}
|
||||
};
|
||||
|
||||
template <class... _Domains>
|
||||
using __indeterminate_domain_t =
|
||||
::cuda::std::_If<sizeof...(_Domains) == 1, decltype((_Domains(), ...)), indeterminate_domain<_Domains...>>;
|
||||
|
||||
template <class _DomainSet>
|
||||
using __domain_from_set_t =
|
||||
::cuda::std::__type_apply<::cuda::std::_If<::cuda::std::__type_set_contains_v<_DomainSet, __not_a_domain>,
|
||||
::cuda::std::__type_always<__not_a_domain>,
|
||||
::cuda::std::__type_quote<__indeterminate_domain_t>>,
|
||||
_DomainSet>;
|
||||
|
||||
template <class... _Domains>
|
||||
using __make_domain_t = __domain_from_set_t<::cuda::std::__make_type_set<_Domains...>>;
|
||||
|
||||
// Common domain for a set of domains
|
||||
template <class... _Domains>
|
||||
struct __common_domain
|
||||
{
|
||||
using type =
|
||||
::cuda::std::__type_call<::cuda::std::__type_try_catch<::cuda::std::__type_quote<::cuda::std::common_type_t>,
|
||||
::cuda::std::__type_quote<__make_domain_t>>,
|
||||
_Domains...>;
|
||||
};
|
||||
|
||||
template <class... _Domains>
|
||||
using __common_domain_t = typename __common_domain<_Domains...>::type;
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
template <class _Tag, class _Sndr, class... _Env>
|
||||
extern __call_result_or_t<get_completion_domain_t<_Tag>, indeterminate_domain<>, env_of_t<_Sndr>, _Env...>
|
||||
__compl_domain_v;
|
||||
|
||||
template <class _Tag, class _Sndr>
|
||||
extern __call_result_or_t<get_completion_domain_t<_Tag>,
|
||||
// If we ask for the completion domain early (without an env)
|
||||
// and it cannot be determined, then:
|
||||
// - if the sender knows it can never complete with _Tag, return
|
||||
// indeterminate_domain<>
|
||||
// - otherwise, return __not_a_domain (indicating that the
|
||||
// completion domain may only be knowable later, when an env
|
||||
// is available)
|
||||
::cuda::std::_If<__never_completes_with<_Sndr, _Tag>, indeterminate_domain<>, __not_a_domain>,
|
||||
env_of_t<_Sndr>>
|
||||
__compl_domain_v<_Tag, _Sndr>;
|
||||
} // namespace __detail
|
||||
|
||||
template <class _Tag, class _Sndr, class... _Env>
|
||||
using __compl_domain_t = decltype(__detail::__compl_domain_v<_Tag, _Sndr, _Env...>);
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
// Specializations of cuda::std::common_type for execution::indeterminate_domain
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
template <class... _Ds, class _Domain>
|
||||
struct common_type<::cuda::experimental::execution::indeterminate_domain<_Ds...>, _Domain>
|
||||
{
|
||||
using type = ::cuda::experimental::execution::__make_domain_t<_Ds..., _Domain>;
|
||||
};
|
||||
|
||||
template <class _Domain, class... _Ds>
|
||||
struct common_type<_Domain, ::cuda::experimental::execution::indeterminate_domain<_Ds...>>
|
||||
{
|
||||
using type = ::cuda::experimental::execution::__make_domain_t<_Ds..., _Domain>;
|
||||
};
|
||||
|
||||
template <class... _As, class... _Bs>
|
||||
struct common_type<::cuda::experimental::execution::indeterminate_domain<_As...>,
|
||||
::cuda::experimental::execution::indeterminate_domain<_Bs...>>
|
||||
{
|
||||
using type = ::cuda::experimental::execution::__make_domain_t<_As..., _Bs...>;
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_DOMAIN
|
||||
@@ -1,352 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX___EXECUTION_ENV_CUH
|
||||
#define __CUDAX___EXECUTION_ENV_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__memory_resource/any_resource.h>
|
||||
#include <cuda/__memory_resource/get_memory_resource.h>
|
||||
#include <cuda/__memory_resource/properties.h>
|
||||
#include <cuda/__stream/get_stream.h>
|
||||
#include <cuda/__type_traits/is_specialization_of.h>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__execution/env.h>
|
||||
#include <cuda/std/__type_traits/remove_cvref.h>
|
||||
#include <cuda/std/__utility/move.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/policy.cuh>
|
||||
#include <cuda/experimental/__execution/queries.cuh>
|
||||
#include <cuda/experimental/__stream/stream_ref.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
namespace execution
|
||||
{
|
||||
template <class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __fwd_env_;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// __env_ref
|
||||
|
||||
//! @brief __env_ref_ is a utility that builds a queryable object from a reference
|
||||
//! to another queryable object.
|
||||
template <class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __env_ref_
|
||||
{
|
||||
_CCCL_TEMPLATE(class _Query, class... _Args)
|
||||
_CCCL_REQUIRES(__queryable_with<_Env, _Query, _Args...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(_Query, _Args&&... __args) const
|
||||
noexcept(__nothrow_queryable_with<_Env, _Query, _Args...>) -> __query_result_t<_Env, _Query, _Args...>
|
||||
{
|
||||
return __env_.query(_Query{}, static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
_Env const& __env_;
|
||||
};
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __env_ref_fn
|
||||
{
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(env<>) const noexcept -> env<>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Env, class = _Env*) // not considered if _Env is a reference type
|
||||
_CCCL_REQUIRES((!::cuda::__is_specialization_of_v<_Env, __fwd_env_>) )
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Env&& __env) const noexcept -> _Env
|
||||
{
|
||||
return static_cast<_Env&&>(__env);
|
||||
}
|
||||
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Env& __env) const noexcept -> __env_ref_<_Env>
|
||||
{
|
||||
return __env_ref_<_Env>{__env};
|
||||
}
|
||||
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(__env_ref_<_Env> __env) const noexcept
|
||||
-> __env_ref_<_Env>
|
||||
{
|
||||
return __env;
|
||||
}
|
||||
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const __fwd_env_<_Env>& __env) const noexcept
|
||||
-> __fwd_env_<_Env const&>
|
||||
{
|
||||
return __fwd_env_<_Env const&>{__env.__env_};
|
||||
}
|
||||
};
|
||||
} // namespace __detail
|
||||
|
||||
template <class _Env>
|
||||
using __env_ref_t _CCCL_NODEBUG_ALIAS = __call_result_t<__detail::__env_ref_fn, _Env>;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT __detail::__env_ref_fn __env_ref{};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// __fwd_env
|
||||
|
||||
//! @brief __fwd_env_ is a utility that forwards queries to a given queryable object
|
||||
//! provided those queries that satisfy the __forwarding_query concept.
|
||||
template <class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __fwd_env_
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Query, class... _Args)
|
||||
_CCCL_REQUIRES(__forwarding_query<_Query> _CCCL_AND __queryable_with<_Env, _Query, _Args...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(_Query, _Args&&... __args) const
|
||||
noexcept(__nothrow_queryable_with<_Env, _Query, _Args...>) -> __query_result_t<_Env, _Query, _Args...>
|
||||
{
|
||||
return __env_.query(_Query{}, static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
_Env __env_;
|
||||
};
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __fwd_env_fn
|
||||
{
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(env<>) const noexcept -> env<>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(__env_ref_<_Env> __env) const noexcept
|
||||
-> __fwd_env_<_Env const&>
|
||||
{
|
||||
return __fwd_env_<_Env const&>{__env.__env_};
|
||||
}
|
||||
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Env&& __env) const noexcept
|
||||
{
|
||||
static_assert(__nothrow_movable<_Env>);
|
||||
// If the environment is already a forwarding environment, we can just return it.
|
||||
if constexpr (__is_specialization_of_v<::cuda::std::remove_cvref_t<_Env>, __fwd_env_>)
|
||||
{
|
||||
return static_cast<_Env&&>(__env);
|
||||
}
|
||||
else
|
||||
{
|
||||
return __fwd_env_<_Env>{static_cast<_Env&&>(__env)};
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace __detail
|
||||
|
||||
template <class _Env>
|
||||
using __fwd_env_t _CCCL_NODEBUG_ALIAS = __call_result_t<__detail::__fwd_env_fn, _Env>;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT __detail::__fwd_env_fn __fwd_env{};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// __sch_env
|
||||
|
||||
//! @brief __sch_env_t is a utility that builds an environment from a scheduler. It
|
||||
//! defines the `get_scheduler` query and provides a default for the `get_domain` query.
|
||||
template <class _Sch>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sch_env_t
|
||||
{
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_scheduler_t) const noexcept -> _Sch
|
||||
{
|
||||
return __sch_;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_domain_t) const noexcept
|
||||
{
|
||||
return __query_result_or_t<_Sch, get_completion_domain_t<set_value_t>, default_domain>{};
|
||||
}
|
||||
|
||||
_Sch __sch_;
|
||||
};
|
||||
|
||||
template <class _Sch>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES __sch_env_t(_Sch) -> __sch_env_t<_Sch>;
|
||||
|
||||
struct __mk_sch_env_t
|
||||
{
|
||||
template <class _Sch, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sch __sch, const _Env&... __env) const noexcept
|
||||
{
|
||||
return __sch_env_t{__call_or(get_completion_scheduler<set_value_t>, __sch, __sch, __env...)};
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT __mk_sch_env_t __mk_sch_env{};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// __sch_attrs
|
||||
|
||||
//! @brief __sch_attrs_t is a utility that builds attributes for a sender from a
|
||||
//! scheduler. It defines the `get_completion_scheduler<set_value_t>` query and provides a default for the
|
||||
//! `get_completion_domain_t<set_value_t>` query.
|
||||
template <class _Sch>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sch_attrs_t
|
||||
{
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_scheduler_t<set_value_t>) const noexcept
|
||||
-> const _Sch&
|
||||
{
|
||||
return __sch_;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class... _Env)
|
||||
_CCCL_REQUIRES(__callable<get_completion_domain_t<set_value_t>, _Sch, _Env...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_domain_t<set_value_t>, const _Env&...) const noexcept
|
||||
{
|
||||
return __call_result_t<get_completion_domain_t<set_value_t>, _Sch, _Env...>{};
|
||||
}
|
||||
|
||||
_Sch __sch_;
|
||||
};
|
||||
|
||||
template <class _Sch>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES __sch_attrs_t(_Sch) -> __sch_attrs_t<_Sch>;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// __inln_attrs
|
||||
|
||||
//! @brief __inln_attrs_t is a utility that builds an attributes queryable for a sender
|
||||
//! that completes inline. It implements get_completion_behavior to return
|
||||
//! completion_behavior::inline_completion, and relies on the logic of
|
||||
//! get_completion_scheduler and get_completion_domain to provide the current scheduler
|
||||
//! and domain based on the environment.
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __inln_attrs_t
|
||||
{
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_behavior_t) const noexcept
|
||||
{
|
||||
return completion_behavior::inline_completion;
|
||||
}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// __join_env
|
||||
namespace __detail
|
||||
{
|
||||
struct __join_env_fn
|
||||
{
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Env&& __env, env<> = {}) const noexcept -> _Env
|
||||
{
|
||||
static_assert(__nothrow_movable<_Env>);
|
||||
return static_cast<_Env&&>(__env);
|
||||
}
|
||||
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(env<>, _Env&& __env) const noexcept -> __fwd_env_t<_Env>
|
||||
{
|
||||
return __fwd_env(static_cast<_Env&&>(__env));
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(env<>, env<>) const noexcept -> env<>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
template <class _First, class _Second>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_First&& __first, _Second&& __second) const noexcept
|
||||
-> env<_First, __fwd_env_t<_Second>>
|
||||
{
|
||||
static_assert(__nothrow_movable<_First>);
|
||||
return {static_cast<_First&&>(__first), __fwd_env(static_cast<_Second&&>(__second))};
|
||||
}
|
||||
};
|
||||
} // namespace __detail
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT __detail::__join_env_fn __join_env{};
|
||||
|
||||
template <class... _Envs>
|
||||
using __join_env_t _CCCL_NODEBUG_ALIAS = __call_result_t<__detail::__join_env_fn, _Envs...>;
|
||||
} // namespace execution
|
||||
|
||||
template <class... _Properties>
|
||||
class env_t
|
||||
{
|
||||
private:
|
||||
using __resource = ::cuda::mr::any_resource<_Properties...>;
|
||||
using __stream_ref = stream_ref;
|
||||
|
||||
__resource __mr_;
|
||||
__stream_ref __stream_ = ::cuda::__invalid_stream();
|
||||
execution::any_execution_policy __policy_ = {};
|
||||
|
||||
public:
|
||||
//! @brief Construct an env_t from an any_resource, a stream and a policy
|
||||
//! @param __mr The any_resource passed in
|
||||
//! @param __stream The stream_ref passed in
|
||||
//! @param __policy The execution_policy passed in
|
||||
_CCCL_HIDE_FROM_ABI env_t(::cuda::mr::any_resource<_Properties...> __mr,
|
||||
__stream_ref __stream = ::cuda::__invalid_stream(),
|
||||
execution::any_execution_policy __policy = {}) noexcept
|
||||
: __mr_(::cuda::std::move(__mr))
|
||||
, __stream_(__stream)
|
||||
, __policy_(__policy)
|
||||
{}
|
||||
|
||||
//! @brief Checks whether another env is compatible with this one. That requires it to have queries for the three
|
||||
//! properties we need
|
||||
template <class _Env>
|
||||
static constexpr bool __is_compatible_env =
|
||||
(::cuda::std::execution::__queryable_with<_Env, ::cuda::mr::get_memory_resource_t>) //
|
||||
&&(::cuda::std::execution::__queryable_with<_Env, ::cuda::get_stream_t>)
|
||||
&& (::cuda::std::execution::__queryable_with<_Env, execution::get_execution_policy_t>);
|
||||
|
||||
//! @brief Construct from an environment that has the right queries
|
||||
//! @param __env The environment we are querying for the required information
|
||||
_CCCL_TEMPLATE(class _Env)
|
||||
_CCCL_REQUIRES((!__same_as<_Env, env_t>) _CCCL_AND __is_compatible_env<_Env>)
|
||||
_CCCL_HIDE_FROM_ABI env_t(const _Env& __env) noexcept
|
||||
: __mr_(__env.query(::cuda::mr::get_memory_resource))
|
||||
, __stream_(__env.query(::cuda::get_stream))
|
||||
, __policy_(__env.query(execution::get_execution_policy))
|
||||
{}
|
||||
|
||||
[[nodiscard]] _CCCL_HIDE_FROM_ABI const __resource& query(::cuda::mr::get_memory_resource_t) const noexcept
|
||||
{
|
||||
return __mr_;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HIDE_FROM_ABI __stream_ref query(::cuda::get_stream_t) const noexcept
|
||||
{
|
||||
return __stream_;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HIDE_FROM_ABI execution::any_execution_policy
|
||||
query(execution::get_execution_policy_t) const noexcept
|
||||
{
|
||||
return __policy_;
|
||||
}
|
||||
};
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif //__CUDAX___EXECUTION_ENV_CUH
|
||||
@@ -1,24 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
// IMPORTANT: This file intentionally lacks a header guard.
|
||||
|
||||
#if !defined(_CUDAX_ASYNC_PROLOGUE_INCLUDED)
|
||||
# error epilogue.cuh included without a prior inclusion of prologue.cuh
|
||||
#endif
|
||||
|
||||
#undef _CUDAX_ASYNC_PROLOGUE_INCLUDED
|
||||
|
||||
#if _CCCL_CUDA_COMPILER(NVHPC)
|
||||
_CCCL_END_NV_DIAG_SUPPRESS()
|
||||
#endif // _CCCL_CUDA_COMPILER(NVHPC)
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
@@ -1,116 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_EXCEPTION
|
||||
#define __CUDAX_EXECUTION_EXCEPTION
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__exception/cuda_error.h>
|
||||
#include <cuda/std/__exception/exception_macros.h>
|
||||
#include <cuda/std/__exception/terminate.h>
|
||||
#include <cuda/std/__utility/move.h>
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
# include <exception> // IWYU pragma: keep
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
// Since there are no exceptions in device code, we provide a stub implementation of
|
||||
// std::exception_ptr and related functions.
|
||||
#if _CCCL_FREESTANDING() || !_CCCL_HOST_COMPILATION()
|
||||
|
||||
struct exception_ptr
|
||||
{
|
||||
private:
|
||||
struct __nullptr_t
|
||||
{};
|
||||
|
||||
//! In libstdc++ and libc++, std::exception_ptr is the size of a pointer, but in MSVC it
|
||||
//! is the size of two pointers. We must match that size here to avoid breaking the ABI
|
||||
//! of any types that contain an exception_ptr.
|
||||
void* __ptrs[1 + _CCCL_COMPILER(MSVC)] = {};
|
||||
|
||||
public:
|
||||
_CCCL_HIDE_FROM_ABI exception_ptr() noexcept = default;
|
||||
|
||||
//! For conversion from nullptr so that code like:
|
||||
//!
|
||||
//! @code
|
||||
//! std::exception_ptr eptr = nullptr;
|
||||
//! @endcode
|
||||
//!
|
||||
//! and
|
||||
//!
|
||||
//! @code
|
||||
//! eptr == nullptr
|
||||
//! @endcode
|
||||
//!
|
||||
//! works as expected.
|
||||
_CCCL_HOST_DEVICE_API constexpr exception_ptr(const __nullptr_t* __ptr) noexcept
|
||||
: exception_ptr()
|
||||
{
|
||||
_CCCL_ASSERT(__ptr == nullptr, "Can only construct exception_ptr from nullptr");
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API explicit constexpr operator bool() const noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool operator!() const noexcept
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
|
||||
operator==(const exception_ptr&, const exception_ptr&) noexcept
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
|
||||
operator!=(const exception_ptr&, const exception_ptr&) noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API inline exception_ptr current_exception() noexcept
|
||||
{
|
||||
return exception_ptr{};
|
||||
}
|
||||
|
||||
[[noreturn]] _CCCL_HOST_DEVICE_API inline void rethrow_exception(const exception_ptr&)
|
||||
{
|
||||
_CCCL_THROW(::cuda::cuda_error, cudaErrorUnknown, "unknown exception");
|
||||
}
|
||||
|
||||
// ^^^ _CCCL_FREESTANDING() || !_CCCL_HOST_COMPILATION() ^^^
|
||||
#else
|
||||
// vvv _CCCL_HOSTED() && _CCCL_HOST_COMPILATION() vvv
|
||||
|
||||
using ::std::current_exception;
|
||||
using ::std::exception_ptr;
|
||||
using ::std::rethrow_exception;
|
||||
|
||||
#endif // _CCCL_HOSTED() && _CCCL_HOST_COMPILATION()
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#endif // __CUDAX_EXECUTION_EXCEPTION
|
||||
@@ -1,337 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_FWD
|
||||
#define __CUDAX_EXECUTION_FWD
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__concepts/same_as.h>
|
||||
#include <cuda/std/__exception/terminate.h>
|
||||
#include <cuda/std/__execution/env.h>
|
||||
#include <cuda/std/__tuple_dir/ignore.h>
|
||||
#include <cuda/std/__type_traits/decay.h>
|
||||
#include <cuda/std/__type_traits/remove_reference.h>
|
||||
#include <cuda/std/__type_traits/type_list.h>
|
||||
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/exception.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
_CCCL_BEGIN_NV_DIAG_SUPPRESS(2642) // call through incomplete class "cuda::experimental::execution::schedule_t"
|
||||
// will always produce an error when instantiated.
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
// so we can refer to the cuda::experimental::__detail namespace below
|
||||
namespace __detail
|
||||
{
|
||||
}
|
||||
namespace execution
|
||||
{
|
||||
namespace __detail
|
||||
{
|
||||
using namespace cuda::experimental::__detail; // NOLINT(misc-unused-using-decls)
|
||||
} // namespace __detail
|
||||
|
||||
// NOLINTBEGIN(misc-unused-using-decls)
|
||||
using ::cuda::std::execution::__forwarding_query;
|
||||
using ::cuda::std::execution::__unwrap_reference_t;
|
||||
using ::cuda::std::execution::env;
|
||||
using ::cuda::std::execution::env_of_t;
|
||||
using ::cuda::std::execution::forwarding_query;
|
||||
using ::cuda::std::execution::forwarding_query_t;
|
||||
using ::cuda::std::execution::get_env;
|
||||
using ::cuda::std::execution::get_env_t;
|
||||
using ::cuda::std::execution::prop;
|
||||
|
||||
using ::cuda::std::execution::__nothrow_queryable_with;
|
||||
using ::cuda::std::execution::__query_result_t;
|
||||
using ::cuda::std::execution::__queryable_with;
|
||||
|
||||
using ::cuda::std::execution::__query_or;
|
||||
using ::cuda::std::execution::__query_result_or_t;
|
||||
// NOLINTEND(misc-unused-using-decls)
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT never_stop_token;
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT inplace_stop_source;
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT inplace_stop_token;
|
||||
|
||||
template <class _Callback>
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT inplace_stop_callback;
|
||||
|
||||
template <class _Token, class _Callback>
|
||||
using stop_callback_for_t _CCCL_NODEBUG_ALIAS = typename _Token::template callback_type<_Callback>;
|
||||
|
||||
template <class _Env, class _Query, bool _Default>
|
||||
_CCCL_CONCEPT __nothrow_queryable_with_or =
|
||||
bool(__queryable_with<_Env, _Query> ? __nothrow_queryable_with<_Env, _Query> : _Default);
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT receiver_t
|
||||
{};
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT operation_state_t
|
||||
{};
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT sender_t
|
||||
{};
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT scheduler_t
|
||||
{};
|
||||
|
||||
template <class _Ty>
|
||||
using __sender_concept_t _CCCL_NODEBUG_ALIAS = typename ::cuda::std::remove_reference_t<_Ty>::sender_concept;
|
||||
|
||||
template <class _Ty>
|
||||
using __receiver_concept_t _CCCL_NODEBUG_ALIAS = typename ::cuda::std::remove_reference_t<_Ty>::receiver_concept;
|
||||
|
||||
template <class _Ty>
|
||||
using __scheduler_concept_t _CCCL_NODEBUG_ALIAS = typename ::cuda::std::remove_reference_t<_Ty>::scheduler_concept;
|
||||
|
||||
template <class _Ty>
|
||||
using __operation_state_concept_t _CCCL_NODEBUG_ALIAS =
|
||||
typename ::cuda::std::remove_reference_t<_Ty>::operation_state_concept;
|
||||
|
||||
template <class _Ty>
|
||||
inline constexpr bool __is_sender = __is_instantiable_with<__sender_concept_t, _Ty>;
|
||||
|
||||
template <class _Ty>
|
||||
inline constexpr bool __is_receiver = __is_instantiable_with<__receiver_concept_t, _Ty>;
|
||||
|
||||
template <class _Ty>
|
||||
inline constexpr bool __is_scheduler = __is_instantiable_with<__scheduler_concept_t, _Ty>;
|
||||
|
||||
template <class _Ty>
|
||||
inline constexpr bool __is_operation_state = __is_instantiable_with<__operation_state_concept_t, _Ty>;
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT dependent_sender_error;
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT default_domain;
|
||||
|
||||
template <class... _Sigs>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT completion_signatures;
|
||||
|
||||
template <class _Sndr, class... _Env>
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto get_completion_signatures();
|
||||
|
||||
template <class _Sndr, class... _Env>
|
||||
using completion_signatures_of_t _CCCL_NODEBUG_ALIAS = decltype(execution::get_completion_signatures<_Sndr, _Env...>());
|
||||
|
||||
#if _CCCL_HAS_CONSTEXPR_EXCEPTIONS()
|
||||
template <class... _What, class... _Values>
|
||||
_CCCL_HOST_DEVICE_API consteval auto invalid_completion_signature(_Values... __values) -> completion_signatures<>;
|
||||
#else // ^^^ _CCCL_HAS_CONSTEXPR_EXCEPTIONS() ^^^ / vvv !_CCCL_HAS_CONSTEXPR_EXCEPTIONS() vvv
|
||||
template <class... _What, class... _Values>
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto invalid_completion_signature(_Values...);
|
||||
#endif // ^^^ !_CCCL_HAS_CONSTEXPR_EXCEPTIONS() ^^^
|
||||
|
||||
// handy enumerations for keeping type names readable
|
||||
enum class __disposition : int8_t
|
||||
{
|
||||
__invalid = -1,
|
||||
__value,
|
||||
__error,
|
||||
__stopped
|
||||
};
|
||||
|
||||
// customization point objects:
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT set_value_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT set_error_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT set_stopped_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT start_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT connect_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT schedule_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT transform_sender_t;
|
||||
|
||||
template <class _Sch>
|
||||
using schedule_result_t _CCCL_NODEBUG_ALIAS = decltype(declval<schedule_t>()(declval<_Sch>()));
|
||||
|
||||
template <class _Sndr, class _Rcvr>
|
||||
using connect_result_t _CCCL_NODEBUG_ALIAS = decltype(declval<connect_t>()(declval<_Sndr>(), declval<_Rcvr>()));
|
||||
|
||||
template <class _Sndr, class _Env>
|
||||
using transform_sender_result_t _CCCL_NODEBUG_ALIAS =
|
||||
decltype(declval<transform_sender_t>()(declval<_Sndr>(), declval<_Env>()));
|
||||
|
||||
template <class _Sndr, class _Rcvr>
|
||||
inline constexpr bool __nothrow_connectable = noexcept(declval<connect_t>()(declval<_Sndr>(), declval<_Rcvr>()));
|
||||
|
||||
// sender factory algorithms:
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT read_env_t;
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT just_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT just_error_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT just_stopped_t;
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT just_from_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT just_error_from_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT just_stopped_from_t;
|
||||
|
||||
// sender adaptor algorithms:
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT let_value_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT let_error_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT let_stopped_t;
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT then_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT upon_error_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT upon_stopped_t;
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT when_all_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT conditional_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT sequence_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT write_env_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT starts_on_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT continues_on_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT on_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT schedule_from_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT bulk_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT bulk_chunked_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT bulk_unchunked_t;
|
||||
|
||||
// sender consumer algorithms:
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT sync_wait_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT start_detached_t;
|
||||
|
||||
// queries:
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT get_allocator_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT get_stop_token_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT get_scheduler_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT get_delegation_scheduler_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT get_forward_progress_guarantee_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT get_available_parallelism_t;
|
||||
template <class _Tag>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT get_completion_scheduler_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT get_domain_t;
|
||||
template <class _Tag>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT get_completion_domain_t;
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT get_completion_behavior_t;
|
||||
|
||||
template <class _Ty>
|
||||
using stop_token_of_t _CCCL_NODEBUG_ALIAS = decay_t<__call_result_t<get_stop_token_t, _Ty>>;
|
||||
|
||||
template <class _Env>
|
||||
using __scheduler_of_t _CCCL_NODEBUG_ALIAS = decay_t<__call_result_t<get_scheduler_t, _Env>>;
|
||||
|
||||
template <class _Env>
|
||||
using __domain_of_t _CCCL_NODEBUG_ALIAS = __call_result_t<get_domain_t, _Env>;
|
||||
|
||||
template <class _Tag, class _Sndr, class... _Env>
|
||||
using __completion_domain_of_t _CCCL_NODEBUG_ALIAS =
|
||||
__call_result_t<get_completion_domain_t<_Tag>, env_of_t<_Sndr>, _Env...>;
|
||||
|
||||
// get_forward_progress_guarantee:
|
||||
enum class forward_progress_guarantee
|
||||
{
|
||||
concurrent,
|
||||
parallel,
|
||||
weakly_parallel
|
||||
};
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
struct __get_tag
|
||||
{
|
||||
template <class _Tag, class... _Child>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(int, _Tag, ::cuda::std::__ignore_t, _Child&&...) const -> _Tag
|
||||
{
|
||||
return _Tag{};
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Sndr, class _Tag = __visit_result_t<__get_tag&, _Sndr, int&>>
|
||||
extern __fn_ptr_t<_Tag> __tag_of_v;
|
||||
} // namespace __detail
|
||||
|
||||
_CCCL_TEMPLATE(class _Sndr)
|
||||
_CCCL_REQUIRES(__is_sender<_Sndr>)
|
||||
using tag_of_t _CCCL_NODEBUG_ALIAS = decltype(__detail::__tag_of_v<_Sndr>());
|
||||
|
||||
template <class _Sndr, class... _Tag>
|
||||
inline constexpr bool __sender_for_v = _CCCL_REQUIRES_EXPR((_Sndr, variadic _Tag))(tag_of_t<_Sndr>{});
|
||||
|
||||
template <class _Sndr, class _Tag>
|
||||
inline constexpr bool __sender_for_v<_Sndr, _Tag> =
|
||||
_CCCL_REQUIRES_EXPR((_Sndr, _Tag))(_Same_as(_Tag) tag_of_t<_Sndr>{});
|
||||
|
||||
template <class _Sndr, class... _Tag>
|
||||
_CCCL_CONCEPT sender_for = __sender_for_v<_Sndr, _Tag...>;
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
template <class _Sig>
|
||||
inline constexpr __disposition __signature_disposition = __disposition::__invalid;
|
||||
template <class... _Ts>
|
||||
inline constexpr __disposition __signature_disposition<set_value_t(_Ts...)> = __disposition::__value;
|
||||
template <class _Ty>
|
||||
inline constexpr __disposition __signature_disposition<set_error_t(_Ty)> = __disposition::__error;
|
||||
template <>
|
||||
inline constexpr __disposition __signature_disposition<set_stopped_t()> = __disposition::__stopped;
|
||||
} // namespace __detail
|
||||
|
||||
struct inline_scheduler;
|
||||
class task_scheduler;
|
||||
|
||||
struct stream_domain;
|
||||
struct stream_context;
|
||||
struct stream_scheduler;
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// __has_completions_for and __never_completes_with
|
||||
template <class _SetTag, class _Sndr, class... _Env>
|
||||
_CCCL_CONCEPT __has_completions_for = _CCCL_REQUIRES_EXPR((_SetTag, _Sndr, variadic _Env)) //
|
||||
( //
|
||||
typename(completion_signatures_of_t<_Sndr, _Env...>),
|
||||
requires(completion_signatures_of_t<_Sndr, _Env...>::count(_SetTag{}) != 0) //
|
||||
);
|
||||
|
||||
template <class _Sndr, class _SetTag, class... _Env>
|
||||
_CCCL_CONCEPT __never_completes_with = _CCCL_REQUIRES_EXPR((_SetTag, _Sndr, variadic _Env)) //
|
||||
( //
|
||||
typename(completion_signatures_of_t<_Sndr, _Env...>),
|
||||
requires(completion_signatures_of_t<_Sndr, _Env...>::count(_SetTag{}) == 0) //
|
||||
);
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// __receiver_archetype
|
||||
template <class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __receiver_archetype
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
template <class... _As>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_value(_As&&...) noexcept;
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_error(_Error&&) noexcept;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_stopped() noexcept;
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> _Env;
|
||||
};
|
||||
} // namespace execution
|
||||
} // namespace cuda::experimental
|
||||
|
||||
_CCCL_END_NV_DIAG_SUPPRESS()
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_FWD
|
||||
@@ -1,345 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_GET_COMPLETION_SIGNATURES
|
||||
#define __CUDAX_EXECUTION_GET_COMPLETION_SIGNATURES
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__type_traits/copy_cvref.h>
|
||||
#include <cuda/std/__type_traits/is_base_of.h>
|
||||
#include <cuda/std/__type_traits/remove_reference.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh> // IWYU pragma: export
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/transform_sender.cuh>
|
||||
|
||||
// include this last:
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
#if __cpp_lib_constexpr_exceptions >= 202502L // constexpr exception types, https://wg21.link/p3378
|
||||
|
||||
using __exception = ::std::exception;
|
||||
|
||||
#elif __cpp_constexpr >= 202411L // constexpr virtual functions
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __exception
|
||||
{
|
||||
_CCCL_HIDE_FROM_ABI constexpr __exception() noexcept = default;
|
||||
_CCCL_HIDE_FROM_ABI virtual constexpr ~__exception() = default;
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API virtual constexpr auto what() const noexcept -> const char*
|
||||
{
|
||||
return "<exception>";
|
||||
}
|
||||
};
|
||||
|
||||
#else // no constexpr virtual functions:
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __exception
|
||||
{
|
||||
_CCCL_HIDE_FROM_ABI constexpr __exception() noexcept = default;
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto what() const noexcept -> const char*
|
||||
{
|
||||
return "<exception>";
|
||||
}
|
||||
};
|
||||
|
||||
#endif // __cpp_lib_constexpr_exceptions >= 202502L
|
||||
|
||||
template <class _Derived>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __compile_time_error : __exception
|
||||
{
|
||||
_CCCL_HIDE_FROM_ABI __compile_time_error() = default;
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto what() const noexcept -> const char*
|
||||
{
|
||||
return static_cast<_Derived const*>(this)->__what();
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Data, class... _What>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sender_type_check_failure //
|
||||
: __compile_time_error<__sender_type_check_failure<_Data, _What...>>
|
||||
{
|
||||
static_assert(__nothrow_movable<_Data>,
|
||||
"The data member of __sender_type_check_failure must be nothrow move constructible.");
|
||||
|
||||
_CCCL_HIDE_FROM_ABI constexpr __sender_type_check_failure() noexcept = default;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __sender_type_check_failure(_Data __data)
|
||||
: __data_(static_cast<_Data&&>(__data))
|
||||
{}
|
||||
|
||||
private:
|
||||
friend struct __compile_time_error<__sender_type_check_failure>;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr auto __what() const noexcept -> const char*
|
||||
{
|
||||
return "This sender is not well-formed. It does not meet the requirements of a sender type.";
|
||||
}
|
||||
|
||||
_Data __data_{};
|
||||
};
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT dependent_sender_error : __compile_time_error<dependent_sender_error>
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit dependent_sender_error(char const* __what) noexcept
|
||||
: __what_(__what)
|
||||
{}
|
||||
|
||||
private:
|
||||
friend struct __compile_time_error<dependent_sender_error>;
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __what() const noexcept -> char const*
|
||||
{
|
||||
return __what_;
|
||||
}
|
||||
|
||||
char const* __what_;
|
||||
};
|
||||
|
||||
template <class _Sndr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __dependent_sender_error : dependent_sender_error
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API constexpr __dependent_sender_error() noexcept
|
||||
: dependent_sender_error{"This sender needs to know its execution " //
|
||||
"environment before it can know how it will complete."}
|
||||
{}
|
||||
|
||||
_CCCL_HOST_DEVICE auto operator+() -> __dependent_sender_error;
|
||||
|
||||
template <class _Ty>
|
||||
_CCCL_HOST_DEVICE auto operator,(_Ty&) -> __dependent_sender_error&;
|
||||
|
||||
template <class... _What>
|
||||
_CCCL_HOST_DEVICE auto operator,(_ERROR<_What...>&) -> _ERROR<_What...>&;
|
||||
};
|
||||
|
||||
// Below is the definition of the _CUDAX_LET_COMPLETIONS portability macro. It
|
||||
// is used to check that an expression's type is a valid completion_signature
|
||||
// specialization.
|
||||
//
|
||||
// USAGE:
|
||||
//
|
||||
// _CUDAX_LET_COMPLETIONS(auto(__cs) = <expression>)
|
||||
// {
|
||||
// // __cs is guaranteed to be a specialization of completion_signatures.
|
||||
// }
|
||||
//
|
||||
// When constexpr exceptions are available (C++26), the macro simply expands to
|
||||
// the moral equivalent of:
|
||||
//
|
||||
// // With constexpr exceptions:
|
||||
// auto __cs = <expression>; // throws if __cs is not a completion_signatures
|
||||
//
|
||||
// When constexpr exceptions are not available, the macro expands to:
|
||||
//
|
||||
// // Without constexpr exceptions:
|
||||
// if constexpr (auto __cs = <expression>; !__valid_completion_signatures<decltype(__cs)>)
|
||||
// {
|
||||
// return __cs;
|
||||
// }
|
||||
// else
|
||||
|
||||
#if _CCCL_HAS_CONSTEXPR_EXCEPTIONS()
|
||||
|
||||
# define _CUDAX_LET_COMPLETIONS(...) \
|
||||
if constexpr ([[maybe_unused]] __VA_ARGS__; false) \
|
||||
{ \
|
||||
} \
|
||||
else
|
||||
|
||||
template <class... _Sndr>
|
||||
[[noreturn, nodiscard]] _CCCL_HOST_DEVICE_API consteval auto __dependent_sender() -> completion_signatures<>
|
||||
{
|
||||
throw __dependent_sender_error<_Sndr...>{};
|
||||
}
|
||||
|
||||
#else // ^^^ _CCCL_HAS_CONSTEXPR_EXCEPTIONS() ^^^ / vvv !_CCCL_HAS_CONSTEXPR_EXCEPTIONS() vvv
|
||||
|
||||
# define _CUDAX_PP_EAT_AUTO_auto(_ID) _ID _CCCL_PP_EAT _CCCL_PP_LPAREN
|
||||
# define _CUDAX_PP_EXPAND_AUTO_auto(_ID) auto _ID
|
||||
# define _CUDAX_LET_COMPLETIONS_ID(...) _CCCL_PP_EXPAND(_CCCL_PP_CAT(_CUDAX_PP_EAT_AUTO_, __VA_ARGS__) _CCCL_PP_RPAREN)
|
||||
|
||||
# define _CUDAX_LET_COMPLETIONS(...) \
|
||||
if constexpr (_CCCL_PP_CAT(_CUDAX_PP_EXPAND_AUTO_, __VA_ARGS__); \
|
||||
!::cuda::experimental::execution::__valid_completion_signatures<decltype(_CUDAX_LET_COMPLETIONS_ID( \
|
||||
__VA_ARGS__))>) \
|
||||
{ \
|
||||
return _CUDAX_LET_COMPLETIONS_ID(__VA_ARGS__); \
|
||||
} \
|
||||
else
|
||||
|
||||
template <class... _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __dependent_sender() -> __dependent_sender_error<_Sndr...>
|
||||
{
|
||||
return __dependent_sender_error<_Sndr...>{};
|
||||
}
|
||||
|
||||
#endif // ^^^ !_CCCL_HAS_CONSTEXPR_EXCEPTIONS() ^^^
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// get_completion_signatures
|
||||
_CCCL_DIAG_PUSH
|
||||
// warning C4913: user defined binary operator ',' exists but no overload could convert all operands,
|
||||
// default built-in binary operator ',' used
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(4913)
|
||||
|
||||
#define _CUDAX_GET_COMPLSIGS(...) \
|
||||
::cuda::std::remove_reference_t<_CCCL_PP_FIRST(__VA_ARGS__)>::template get_completion_signatures<__VA_ARGS__>()
|
||||
|
||||
#define _CUDAX_CHECKED_COMPLSIGS(...) \
|
||||
(static_cast<void>(__VA_ARGS__), void(), execution::__checked_complsigs<decltype(__VA_ARGS__)>())
|
||||
|
||||
struct _A_GET_COMPLETION_SIGNATURES_CUSTOMIZATION_RETURNED_A_TYPE_THAT_IS_NOT_A_COMPLETION_SIGNATURES_SPECIALIZATION
|
||||
{};
|
||||
|
||||
template <class _Completions>
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __checked_complsigs()
|
||||
{
|
||||
_CUDAX_LET_COMPLETIONS(auto(__cs) = _Completions())
|
||||
{
|
||||
if constexpr (__valid_completion_signatures<_Completions>)
|
||||
{
|
||||
return __cs;
|
||||
}
|
||||
else
|
||||
{
|
||||
return invalid_completion_signature<
|
||||
_A_GET_COMPLETION_SIGNATURES_CUSTOMIZATION_RETURNED_A_TYPE_THAT_IS_NOT_A_COMPLETION_SIGNATURES_SPECIALIZATION,
|
||||
_WITH_SIGNATURES(_Completions)>();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Sndr, class... _Env>
|
||||
using __get_complsigs_t = decltype(_CUDAX_GET_COMPLSIGS(_Sndr, _Env...));
|
||||
|
||||
template <class _Sndr, class... _Env>
|
||||
inline constexpr bool __has_get_completion_signatures = false;
|
||||
|
||||
// clang-format off
|
||||
template <class _Sndr>
|
||||
inline constexpr bool __has_get_completion_signatures<_Sndr> =
|
||||
_CCCL_REQUIRES_EXPR((_Sndr))
|
||||
(
|
||||
typename(__get_complsigs_t<_Sndr>)
|
||||
);
|
||||
|
||||
template <class _Sndr, class _Env>
|
||||
inline constexpr bool __has_get_completion_signatures<_Sndr, _Env> =
|
||||
_CCCL_REQUIRES_EXPR((_Sndr, _Env))
|
||||
(
|
||||
typename(__get_complsigs_t<_Sndr, _Env>)
|
||||
);
|
||||
// clang-format on
|
||||
|
||||
struct _COULD_NOT_DETERMINE_COMPLETION_SIGNATURES_FOR_THIS_SENDER
|
||||
{};
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sndr, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __get_completion_signatures_helper()
|
||||
{
|
||||
if constexpr (__has_get_completion_signatures<_Sndr, _Env...>)
|
||||
{
|
||||
return _CUDAX_CHECKED_COMPLSIGS(_CUDAX_GET_COMPLSIGS(_Sndr, _Env...));
|
||||
}
|
||||
else if constexpr (__has_get_completion_signatures<_Sndr>)
|
||||
{
|
||||
return _CUDAX_CHECKED_COMPLSIGS(_CUDAX_GET_COMPLSIGS(_Sndr));
|
||||
}
|
||||
// else if constexpr (__is_awaitable<_Sndr, __env_promise<_Env>...>)
|
||||
// {
|
||||
// using Result _CCCL_NODEBUG_ALIAS = __await_result_t<_Sndr, __env_promise<_Env>...>;
|
||||
// return completion_signatures{__set_value_v<Result>, __set_error_v<>, __set_stopped_v};
|
||||
// }
|
||||
else if constexpr (sizeof...(_Env) == 0)
|
||||
{
|
||||
return __dependent_sender<_Sndr>();
|
||||
}
|
||||
else
|
||||
{
|
||||
return invalid_completion_signature<_COULD_NOT_DETERMINE_COMPLETION_SIGNATURES_FOR_THIS_SENDER,
|
||||
_WITH_SENDER(_Sndr),
|
||||
_WITH_ENVIRONMENT(_Env...)>();
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Sndr, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
static_assert(sizeof...(_Env) <= 1, "At most one environment is allowed.");
|
||||
if constexpr (0 == sizeof...(_Env))
|
||||
{
|
||||
return execution::__get_completion_signatures_helper<_Sndr>();
|
||||
}
|
||||
else
|
||||
{
|
||||
// Apply a lazy sender transform if one exists before computing the completion signatures:
|
||||
using __new_sndr_t = __call_result_t<transform_sender_t, _Sndr, _Env...>;
|
||||
return execution::__get_completion_signatures_helper<__new_sndr_t, _Env...>();
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Parent, class _Child, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto get_child_completion_signatures()
|
||||
{
|
||||
return get_completion_signatures<::cuda::std::__copy_cvref_t<_Parent, _Child>, __fwd_env_t<_Env>...>();
|
||||
}
|
||||
|
||||
#undef _CUDAX_GET_COMPLSIGS
|
||||
#undef _CUDAX_CHECKED_COMPLSIGS
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
#if _CCCL_HAS_CONSTEXPR_EXCEPTIONS()
|
||||
// When asked for its completions without an envitonment, a dependent sender
|
||||
// will throw an exception of a type derived from `dependent_sender_error`.
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API consteval bool __is_dependent_sender() noexcept
|
||||
try
|
||||
{
|
||||
(void) get_completion_signatures<_Sndr>();
|
||||
return false; // didn't throw, not a dependent sender
|
||||
}
|
||||
catch (dependent_sender_error&)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
return false; // different kind of exception was thrown; not a dependent sender
|
||||
}
|
||||
#else // ^^^ _CCCL_HAS_CONSTEXPR_EXCEPTIONS() ^^^ / vvv !_CCCL_HAS_CONSTEXPR_EXCEPTIONS() vvv
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __is_dependent_sender() noexcept -> bool
|
||||
{
|
||||
using _Completions _CCCL_NODEBUG_ALIAS = decltype(get_completion_signatures<_Sndr>());
|
||||
return ::cuda::std::is_base_of_v<dependent_sender_error, _Completions>;
|
||||
}
|
||||
#endif // ^^^ !_CCCL_HAS_CONSTEXPR_EXCEPTIONS() ^^^
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_GET_COMPLETION_SIGNATURES
|
||||
@@ -1,102 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_INLINE_SCHEDULER
|
||||
#define __CUDAX_EXECUTION_INLINE_SCHEDULER
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__utility/immovable.h>
|
||||
|
||||
#include <cuda/experimental/__execution/completion_behavior.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/domain.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
//! Scheduler that returns a sender that always completes inline (successfully).
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT inline_scheduler : __inln_attrs_t
|
||||
{
|
||||
private:
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __attrs_t : __inln_attrs_t
|
||||
{};
|
||||
|
||||
template <class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t : __immovable
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void start() noexcept
|
||||
{
|
||||
set_value(static_cast<_Rcvr&&>(__rcvr));
|
||||
}
|
||||
|
||||
_Rcvr __rcvr;
|
||||
};
|
||||
|
||||
public:
|
||||
using scheduler_concept = scheduler_t;
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class Self>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto get_completion_signatures() noexcept
|
||||
{
|
||||
return completion_signatures<set_value_t()>{};
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) const noexcept -> __opstate_t<_Rcvr>
|
||||
{
|
||||
return {{}, static_cast<_Rcvr&&>(__rcvr)};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto get_env() noexcept -> __attrs_t
|
||||
{
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto schedule() const noexcept -> __sndr_t
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(inline_scheduler, inline_scheduler) noexcept
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator!=(inline_scheduler, inline_scheduler) noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_INLINE_SCHEDULER
|
||||
@@ -1,295 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
// Copyright (c) 2021-2022 Facebook, Inc & AFFILIATES.
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_INTRUSIVE_QUEUE
|
||||
#define __CUDAX_EXECUTION_INTRUSIVE_QUEUE
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__iterator/iterator_traits.h>
|
||||
#include <cuda/std/__utility/exchange.h>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
template <auto _Next>
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT __intrusive_queue;
|
||||
|
||||
template <class _Item, _Item* _Item::* _Next>
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT __intrusive_queue<_Next>
|
||||
{
|
||||
public:
|
||||
_CCCL_HIDE_FROM_ABI __intrusive_queue() noexcept = default;
|
||||
|
||||
_CCCL_HOST_DEVICE_API __intrusive_queue(__intrusive_queue&& __other) noexcept
|
||||
: __head_(::cuda::std::exchange(__other.__head_, nullptr))
|
||||
, __tail_(::cuda::std::exchange(__other.__tail_, nullptr))
|
||||
{}
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto operator=(__intrusive_queue&& __other) noexcept -> __intrusive_queue&
|
||||
{
|
||||
__head_ = ::cuda::std::exchange(__other.__head_, nullptr);
|
||||
__tail_ = ::cuda::std::exchange(__other.__tail_, nullptr);
|
||||
return *this;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API ~__intrusive_queue()
|
||||
{
|
||||
_CCCL_ASSERT(empty(), "");
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API static auto make_reversed(_Item* __list) noexcept -> __intrusive_queue
|
||||
{
|
||||
_Item* __new_head = nullptr;
|
||||
_Item* __new_tail = __list;
|
||||
|
||||
while (__list != nullptr)
|
||||
{
|
||||
_Item* __next = __list->*_Next;
|
||||
__list->*_Next = __new_head;
|
||||
__new_head = __list;
|
||||
__list = __next;
|
||||
}
|
||||
|
||||
__intrusive_queue __result;
|
||||
__result.__head_ = __new_head;
|
||||
__result.__tail_ = __new_tail;
|
||||
return __result;
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API static auto make(_Item* __list) noexcept -> __intrusive_queue
|
||||
{
|
||||
__intrusive_queue __result{};
|
||||
__result.__head_ = __list;
|
||||
__result.__tail_ = __list;
|
||||
if (__list == nullptr)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
while (__result.__tail_->*_Next != nullptr)
|
||||
{
|
||||
__result.__tail_ = __result.__tail_->*_Next;
|
||||
}
|
||||
return __result;
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API auto empty() const noexcept -> bool
|
||||
{
|
||||
return __head_ == nullptr;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void clear() noexcept
|
||||
{
|
||||
__head_ = nullptr;
|
||||
__tail_ = nullptr;
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API auto pop_front() noexcept -> _Item*
|
||||
{
|
||||
_CCCL_ASSERT(!empty(), "");
|
||||
_Item* __item = ::cuda::std::exchange(__head_, __head_->*_Next);
|
||||
// This should test if __head_ == nullptr, but due to a bug in
|
||||
// nvc++'s optimization, `__head_` isn't assigned until later.
|
||||
// Filed as NVBug#3952534.
|
||||
if (__item->*_Next == nullptr)
|
||||
{
|
||||
__tail_ = nullptr;
|
||||
}
|
||||
return __item;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void push_front(_Item* __item) noexcept
|
||||
{
|
||||
_CCCL_ASSERT(__item != nullptr, "");
|
||||
__item->*_Next = __head_;
|
||||
__head_ = __item;
|
||||
if (__tail_ == nullptr)
|
||||
{
|
||||
__tail_ = __item;
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void push_back(_Item* __item) noexcept
|
||||
{
|
||||
_CCCL_ASSERT(__item != nullptr, "");
|
||||
__item->*_Next = nullptr;
|
||||
(empty() ? __head_ : __tail_->*_Next) = __item;
|
||||
__tail_ = __item;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void append(__intrusive_queue __other) noexcept
|
||||
{
|
||||
if (!__other.empty())
|
||||
{
|
||||
(empty() ? __head_ : __tail_->*_Next) = ::cuda::std::exchange(__other.__head_, nullptr);
|
||||
__tail_ = ::cuda::std::exchange(__other.__tail_, nullptr);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void prepend(__intrusive_queue __other) noexcept
|
||||
{
|
||||
if (!__other.empty())
|
||||
{
|
||||
__other.__tail_->*_Next = __head_;
|
||||
__head_ = __other.__head_;
|
||||
if (__tail_ == nullptr)
|
||||
{
|
||||
__tail_ = __other.__tail_;
|
||||
}
|
||||
|
||||
__other.clear();
|
||||
}
|
||||
}
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT iterator
|
||||
{
|
||||
using value_type _CCCL_NODEBUG_ALIAS = _Item*;
|
||||
using difference_type _CCCL_NODEBUG_ALIAS = ::cuda::std::ptrdiff_t;
|
||||
using pointer _CCCL_NODEBUG_ALIAS = _Item* const*;
|
||||
using reference _CCCL_NODEBUG_ALIAS = _Item* const&;
|
||||
using iterator_category _CCCL_NODEBUG_ALIAS = ::cuda::std::forward_iterator_tag;
|
||||
|
||||
_CCCL_HIDE_FROM_ABI iterator() noexcept = default;
|
||||
|
||||
_CCCL_HOST_DEVICE_API explicit iterator(_Item* __pred, _Item* __item) noexcept
|
||||
: __predecessor_(__pred)
|
||||
, __item_(__item)
|
||||
{}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API auto operator*() const noexcept -> _Item* const&
|
||||
{
|
||||
_CCCL_ASSERT(__item_ != nullptr, "");
|
||||
return __item_;
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API auto operator->() const noexcept -> _Item* const*
|
||||
{
|
||||
_CCCL_ASSERT(__item_ != nullptr, "");
|
||||
return &__item_;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto operator++() noexcept -> iterator&
|
||||
{
|
||||
_CCCL_ASSERT(__item_ != nullptr, "");
|
||||
__predecessor_ = ::cuda::std::exchange(__item_, __item_->*_Next);
|
||||
return *this;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto operator++(int) noexcept -> iterator
|
||||
{
|
||||
iterator __result = *this;
|
||||
++*this;
|
||||
return __result;
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API friend auto operator==(const iterator& __lhs, const iterator& __rhs) noexcept -> bool
|
||||
{
|
||||
return __lhs.__item_ == __rhs.__item_;
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API friend auto operator!=(const iterator& __lhs, const iterator& __rhs) noexcept -> bool
|
||||
{
|
||||
return __lhs.__item_ != __rhs.__item_;
|
||||
}
|
||||
|
||||
_Item* __predecessor_ = nullptr;
|
||||
_Item* __item_ = nullptr;
|
||||
};
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API auto begin() const noexcept -> iterator
|
||||
{
|
||||
return iterator(nullptr, __head_);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API auto end() const noexcept -> iterator
|
||||
{
|
||||
return iterator(__tail_, nullptr);
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void splice(iterator pos, __intrusive_queue& other, iterator first, iterator last) noexcept
|
||||
{
|
||||
if (first == last)
|
||||
{
|
||||
return;
|
||||
}
|
||||
_CCCL_ASSERT(first.__item_ != nullptr, "");
|
||||
_CCCL_ASSERT(last.__predecessor_ != nullptr, "");
|
||||
if (other.__head_ == first.__item_)
|
||||
{
|
||||
other.__head_ = last.__item_;
|
||||
if (other.__head_ == nullptr)
|
||||
{
|
||||
other.__tail_ = nullptr;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_ASSERT(first.__predecessor_ != nullptr, "");
|
||||
first.__predecessor_->*_Next = last.__item_;
|
||||
last.__predecessor_->*_Next = pos.__item_;
|
||||
}
|
||||
if (empty())
|
||||
{
|
||||
__head_ = first.__item_;
|
||||
__tail_ = last.__predecessor_;
|
||||
}
|
||||
else
|
||||
{
|
||||
pos.__predecessor_->*_Next = first.__item_;
|
||||
if (pos.__item_ == nullptr)
|
||||
{
|
||||
__tail_ = last.__predecessor_;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto front() const noexcept -> _Item* const&
|
||||
{
|
||||
return __head_;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto back() const noexcept -> _Item* const&
|
||||
{
|
||||
return __tail_;
|
||||
}
|
||||
|
||||
private:
|
||||
_CCCL_HOST_DEVICE_API explicit __intrusive_queue(_Item* __head, _Item* __tail) noexcept
|
||||
: __head_(__head)
|
||||
, __tail_(__tail)
|
||||
{}
|
||||
|
||||
_Item* __head_ = nullptr;
|
||||
_Item* __tail_ = nullptr;
|
||||
};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_INTRUSIVE_QUEUE
|
||||
@@ -1,182 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_JUST
|
||||
#define __CUDAX_EXECUTION_JUST
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__utility/pod_tuple.h>
|
||||
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
template <class _JustTag, class _SetTag>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __just_t
|
||||
{
|
||||
private:
|
||||
friend struct just_t;
|
||||
friend struct just_error_t;
|
||||
friend struct just_stopped_t;
|
||||
|
||||
using __just_tag_t = _JustTag;
|
||||
using __set_tag_t = _SetTag;
|
||||
|
||||
template <class _Rcvr, class... _Ts>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
using __tuple_t = ::cuda::std::__tuple<_Ts...>;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __opstate_t(_Rcvr&& __rcvr, __tuple_t __values)
|
||||
: __rcvr_{static_cast<_Rcvr&&>(__rcvr)}
|
||||
, __values_{static_cast<__tuple_t&&>(__values)}
|
||||
{}
|
||||
|
||||
#if !_CCCL_COMPILER(GCC)
|
||||
// Because of gcc#98995, making this operation state immovable will cause errors in
|
||||
// functions that return composite operation states by value. Fortunately, the `just`
|
||||
// operation state doesn't strictly need to be immovable, since its address never
|
||||
// escapes. So for gcc, we let this operation state be movable.
|
||||
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98995
|
||||
_CCCL_IMMOVABLE(__opstate_t);
|
||||
#endif // !_CCCL_COMPILER(GCC)
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void start() noexcept
|
||||
{
|
||||
::cuda::std::__apply(
|
||||
_SetTag{}, static_cast<::cuda::std::__tuple<_Ts...>&&>(__values_), static_cast<_Rcvr&&>(__rcvr_));
|
||||
}
|
||||
|
||||
_Rcvr __rcvr_;
|
||||
__tuple_t __values_;
|
||||
};
|
||||
|
||||
template <class... _Ts>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_base_t;
|
||||
|
||||
public:
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Ts... __ts) const;
|
||||
};
|
||||
|
||||
struct just_t : __just_t<just_t, set_value_t>
|
||||
{
|
||||
template <class... _Ts>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
};
|
||||
|
||||
struct just_error_t : __just_t<just_error_t, set_error_t>
|
||||
{
|
||||
template <class... _Ts>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
};
|
||||
|
||||
struct just_stopped_t : __just_t<just_stopped_t, set_stopped_t>
|
||||
{
|
||||
template <class... _Ts>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
};
|
||||
|
||||
template <class _JustTag, class _SetTag>
|
||||
template <class... _Ts>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __just_t<_JustTag, _SetTag>::__sndr_base_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures() noexcept
|
||||
{
|
||||
return completion_signatures<__set_tag_t(_Ts...)>{};
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
connect(_Rcvr __rcvr) && noexcept(__nothrow_decay_copyable<_Rcvr, _Ts...>) -> __opstate_t<_Rcvr, _Ts...>
|
||||
{
|
||||
return __opstate_t<_Rcvr, _Ts...>{
|
||||
static_cast<_Rcvr&&>(__rcvr), static_cast<::cuda::std::__tuple<_Ts...>&&>(__values_)};
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
connect(_Rcvr __rcvr) const& noexcept(__nothrow_decay_copyable<_Rcvr, _Ts const&...>) -> __opstate_t<_Rcvr, _Ts...>
|
||||
{
|
||||
return __opstate_t<_Rcvr, _Ts...>{static_cast<_Rcvr&&>(__rcvr), __values_};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto get_env() noexcept
|
||||
{
|
||||
return __inln_attrs_t{};
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ __just_tag_t __tag_;
|
||||
::cuda::std::__tuple<_Ts...> __values_;
|
||||
};
|
||||
|
||||
template <class... _Ts>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT just_t::__sndr_t : __just_t<just_t, set_value_t>::__sndr_base_t<_Ts...>
|
||||
{};
|
||||
|
||||
template <class... _Ts>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT just_error_t::__sndr_t : __just_t<just_error_t, set_error_t>::__sndr_base_t<_Ts...>
|
||||
{
|
||||
static_assert(sizeof...(_Ts) == 1, "just_error_t must be called with exactly one error type.");
|
||||
};
|
||||
|
||||
template <class... _Ts>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT just_stopped_t::__sndr_t
|
||||
: __just_t<just_stopped_t, set_stopped_t>::__sndr_base_t<_Ts...>
|
||||
{
|
||||
static_assert(sizeof...(_Ts) == 0, "just_stopped_t must not be called with any types.");
|
||||
};
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _JustTag, class _SetTag>
|
||||
template <class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto __just_t<_JustTag, _SetTag>::operator()(_Ts... __ts) const
|
||||
{
|
||||
using __sndr_t = typename _JustTag::template __sndr_t<_Ts...>;
|
||||
return __sndr_t{{{}, {static_cast<_Ts&&>(__ts)...}}};
|
||||
}
|
||||
|
||||
template <class... _Ts>
|
||||
inline constexpr int structured_binding_size<just_t::__sndr_t<_Ts...>> = 2;
|
||||
template <class... _Ts>
|
||||
inline constexpr int structured_binding_size<just_error_t::__sndr_t<_Ts...>> = 2;
|
||||
template <class... _Ts>
|
||||
inline constexpr int structured_binding_size<just_stopped_t::__sndr_t<_Ts...>> = 2;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto just = just_t{};
|
||||
_CCCL_GLOBAL_CONSTANT auto just_error = just_error_t{};
|
||||
_CCCL_GLOBAL_CONSTANT auto just_stopped = just_stopped_t{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_JUST
|
||||
@@ -1,198 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_JUST_FROM
|
||||
#define __CUDAX_EXECUTION_JUST_FROM
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__cccl/unreachable.h>
|
||||
#include <cuda/std/__type_traits/conditional.h>
|
||||
#include <cuda/std/__utility/pod_tuple.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/transform_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
struct _AN_ERROR_COMPLETION_MUST_HAVE_EXACTLY_ONE_ERROR_ARGUMENT;
|
||||
struct _A_STOPPED_COMPLETION_MUST_HAVE_NO_ARGUMENTS;
|
||||
|
||||
template <class _JustFromTag, class _SetTag>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __just_from_t
|
||||
{
|
||||
_CUDAX_SEMI_PRIVATE:
|
||||
friend struct just_from_t;
|
||||
friend struct just_error_from_t;
|
||||
friend struct just_stopped_from_t;
|
||||
|
||||
using __just_from_tag_t = _JustFromTag;
|
||||
|
||||
using __diag_t _CCCL_NODEBUG_ALIAS =
|
||||
::cuda::std::conditional_t<_SetTag{} == set_error,
|
||||
_AN_ERROR_COMPLETION_MUST_HAVE_EXACTLY_ONE_ERROR_ARGUMENT,
|
||||
_A_STOPPED_COMPLETION_MUST_HAVE_NO_ARGUMENTS>;
|
||||
|
||||
template <class... _Ts>
|
||||
using __error_t _CCCL_NODEBUG_ALIAS =
|
||||
_ERROR<_WHERE(_IN_ALGORITHM, _JustFromTag), _WHAT(__diag_t), _WITH_COMPLETION_SIGNATURE<_SetTag(_Ts...)>>;
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __probe_fn
|
||||
{
|
||||
template <class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API auto operator()(_Ts&&... __ts) const noexcept
|
||||
-> ::cuda::std::_If<__detail::__signature_disposition<_SetTag(_Ts...)> != __disposition::__invalid,
|
||||
completion_signatures<_SetTag(_Ts...)>,
|
||||
__error_t<_Ts...>>;
|
||||
};
|
||||
|
||||
template <class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __complete_fn
|
||||
{
|
||||
template <class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API void operator()(_Ts&&... __ts) const noexcept
|
||||
{
|
||||
_SetTag{}(static_cast<_Rcvr&&>(__rcvr_), static_cast<_Ts&&>(__ts)...);
|
||||
}
|
||||
|
||||
_Rcvr& __rcvr_;
|
||||
};
|
||||
|
||||
template <class _Rcvr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_HOST_DEVICE_API constexpr void start() noexcept
|
||||
{
|
||||
static_cast<_Fn&&>(__fn_)(__complete_fn<_Rcvr>{__rcvr_});
|
||||
}
|
||||
|
||||
_Rcvr __rcvr_;
|
||||
_Fn __fn_;
|
||||
};
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_base_t;
|
||||
|
||||
public:
|
||||
template <class _Fn>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Fn __fn) const noexcept;
|
||||
};
|
||||
|
||||
struct just_from_t : __just_from_t<just_from_t, set_value_t>
|
||||
{
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
};
|
||||
|
||||
struct just_error_from_t : __just_from_t<just_error_from_t, set_error_t>
|
||||
{
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
};
|
||||
|
||||
struct just_stopped_from_t : __just_from_t<just_stopped_from_t, set_stopped_t>
|
||||
{
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
};
|
||||
|
||||
template <class _JustFromTag, class _SetTag>
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __just_from_t<_JustFromTag, _SetTag>::__sndr_base_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class _Self, class...>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures() noexcept
|
||||
{
|
||||
return __call_result_t<_Fn, __probe_fn>{};
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) && //
|
||||
noexcept(__nothrow_decay_copyable<_Rcvr, _Fn>) -> __opstate_t<_Rcvr, _Fn>
|
||||
{
|
||||
return __opstate_t<_Rcvr, _Fn>{static_cast<_Rcvr&&>(__rcvr), static_cast<_Fn&&>(__fn_)};
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) const& //
|
||||
noexcept(__nothrow_decay_copyable<_Rcvr, _Fn const&>) -> __opstate_t<_Rcvr, _Fn>
|
||||
{
|
||||
return __opstate_t<_Rcvr, _Fn>{static_cast<_Rcvr&&>(__rcvr), __fn_};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept
|
||||
{
|
||||
return __inln_attrs_t{};
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ __just_from_tag_t __tag_;
|
||||
_Fn __fn_;
|
||||
};
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT just_from_t::__sndr_t : __just_from_t<just_t, set_value_t>::__sndr_base_t<_Fn>
|
||||
{};
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT just_error_from_t::__sndr_t
|
||||
: __just_from_t<just_error_t, set_error_t>::__sndr_base_t<_Fn>
|
||||
{};
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT just_stopped_from_t::__sndr_t
|
||||
: __just_from_t<just_stopped_t, set_stopped_t>::__sndr_base_t<_Fn>
|
||||
{};
|
||||
|
||||
template <class _JustFromTag, class _SetTag>
|
||||
template <class _Fn>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto __just_from_t<_JustFromTag, _SetTag>::operator()(_Fn __fn) const noexcept
|
||||
{
|
||||
using __sndr_t = typename _JustFromTag::template __sndr_t<_Fn>;
|
||||
using __completions _CCCL_NODEBUG_ALIAS = __call_result_t<_Fn, __probe_fn>;
|
||||
static_assert(__valid_completion_signatures<__completions>,
|
||||
"The function passed to just_from must return an instance of a specialization of "
|
||||
"completion_signatures<>.");
|
||||
return __sndr_t{{{}, static_cast<_Fn&&>(__fn)}};
|
||||
}
|
||||
|
||||
template <class _Fn>
|
||||
inline constexpr int structured_binding_size<just_from_t::__sndr_t<_Fn>> = 2;
|
||||
template <class _Fn>
|
||||
inline constexpr int structured_binding_size<just_error_from_t::__sndr_t<_Fn>> = 2;
|
||||
template <class _Fn>
|
||||
inline constexpr int structured_binding_size<just_stopped_from_t::__sndr_t<_Fn>> = 2;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto just_from = just_from_t{};
|
||||
_CCCL_GLOBAL_CONSTANT auto just_error_from = just_error_from_t{};
|
||||
_CCCL_GLOBAL_CONSTANT auto just_stopped_from = just_stopped_from_t{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_JUST_FROM
|
||||
@@ -1,135 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_LAZY
|
||||
#define __CUDAX_EXECUTION_LAZY
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__cstddef/byte.h>
|
||||
#include <cuda/std/__memory/addressof.h>
|
||||
#include <cuda/std/__memory/construct_at.h>
|
||||
#include <cuda/std/__new/device_new.h>
|
||||
#include <cuda/std/__new/launder.h>
|
||||
#include <cuda/std/__type_traits/copy_cvref.h>
|
||||
#include <cuda/std/__utility/integer_sequence.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/meta.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
#include <cuda/experimental/__utility/manual_lifetime.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
/// @brief A lazy type that can be used to delay the construction of a type.
|
||||
template <class _Ty>
|
||||
using __lazy = ::cuda::experimental::__manual_lifetime<_Ty>;
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
template <size_t _Idx, size_t _Size, size_t _Align>
|
||||
struct __lazy_box_
|
||||
{
|
||||
static_assert(_Size != 0);
|
||||
alignas(_Align)::cuda::std::byte __data_[_Size];
|
||||
};
|
||||
|
||||
template <size_t _Idx, class _Ty>
|
||||
using __lazy_box _CCCL_NODEBUG_ALIAS = __lazy_box_<_Idx, sizeof(_Ty), alignof(_Ty)>;
|
||||
} // namespace __detail
|
||||
|
||||
template <class _Idx, class... _Ts>
|
||||
struct __lazy_tupl;
|
||||
|
||||
template <>
|
||||
struct __lazy_tupl<::cuda::std::index_sequence<>>
|
||||
{
|
||||
template <class _Fn, class _Self, class... _Us>
|
||||
_CCCL_HOST_DEVICE_API static auto __apply(_Fn&& __fn, _Self&&, _Us&&... __us) //
|
||||
noexcept(__nothrow_callable<_Fn, _Us...>) -> __call_result_t<_Fn, _Us...>
|
||||
{
|
||||
return static_cast<_Fn&&>(__fn)(static_cast<_Us&&>(__us)...);
|
||||
}
|
||||
};
|
||||
|
||||
template <size_t... _Idx, class... _Ts>
|
||||
struct __lazy_tupl<::cuda::std::index_sequence<_Idx...>, _Ts...> : __detail::__lazy_box<_Idx, _Ts>...
|
||||
{
|
||||
template <size_t _Ny>
|
||||
using __at _CCCL_NODEBUG_ALIAS = ::cuda::std::__type_index_c<_Ny, _Ts...>;
|
||||
|
||||
_CCCL_HOST_DEVICE_API __lazy_tupl() noexcept {}
|
||||
|
||||
_CCCL_HOST_DEVICE_API ~__lazy_tupl()
|
||||
{
|
||||
((__engaged_[_Idx] ? ::cuda::std::destroy_at(__get<_Idx, _Ts>()) : void(0)), ...);
|
||||
}
|
||||
|
||||
template <size_t _Ny, class _Ty>
|
||||
_CCCL_HOST_DEVICE_API _Ty* __get() noexcept
|
||||
{
|
||||
return reinterpret_cast<_Ty*>(this->__detail::__lazy_box<_Ny, _Ty>::__data_);
|
||||
}
|
||||
|
||||
template <size_t _Ny, class... _Us>
|
||||
_CCCL_HOST_DEVICE_API __at<_Ny>& __emplace(_Us&&... __us) //
|
||||
noexcept(__nothrow_constructible<__at<_Ny>, _Us...>)
|
||||
{
|
||||
using _Ty _CCCL_NODEBUG_ALIAS = __at<_Ny>;
|
||||
_Ty* __value_ = ::new (static_cast<void*>(__get<_Ny, _Ty>())) _Ty{static_cast<_Us&&>(__us)...};
|
||||
__engaged_[_Ny] = true;
|
||||
return *::cuda::std::launder(__value_);
|
||||
}
|
||||
|
||||
template <class _Fn, class _Self, class... _Us>
|
||||
_CCCL_HOST_DEVICE_API static auto __apply(_Fn&& __fn, _Self&& __self, _Us&&... __us) //
|
||||
noexcept(__nothrow_callable<_Fn, _Us..., ::cuda::std::__copy_cvref_t<_Self, _Ts>...>)
|
||||
-> __call_result_t<_Fn, _Us..., ::cuda::std::__copy_cvref_t<_Self, _Ts>...>
|
||||
{
|
||||
return static_cast<_Fn&&>(
|
||||
__fn)(static_cast<_Us&&>(__us)...,
|
||||
static_cast<::cuda::std::__copy_cvref_t<_Self, _Ts>&&>(*__self.template __get<_Idx, _Ts>())...);
|
||||
}
|
||||
|
||||
bool __engaged_[sizeof...(_Ts)] = {};
|
||||
};
|
||||
|
||||
#if _CCCL_COMPILER(MSVC)
|
||||
template <class... _Ts>
|
||||
struct __mk_lazy_tuple_
|
||||
{
|
||||
using __indices_t _CCCL_NODEBUG_ALIAS = ::cuda::std::make_index_sequence<sizeof...(_Ts)>;
|
||||
using type _CCCL_NODEBUG_ALIAS = __lazy_tupl<__indices_t, _Ts...>;
|
||||
};
|
||||
|
||||
template <class... _Ts>
|
||||
using __lazy_tuple _CCCL_NODEBUG_ALIAS = typename __mk_lazy_tuple_<_Ts...>::type;
|
||||
#else // ^^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
|
||||
template <class... _Ts>
|
||||
using __lazy_tuple _CCCL_NODEBUG_ALIAS = __lazy_tupl<::cuda::std::make_index_sequence<sizeof...(_Ts)>, _Ts...>;
|
||||
#endif // !_CCCL_COMPILER(MSVC)
|
||||
|
||||
template <class... _Ts>
|
||||
using __decayed_lazy_tuple _CCCL_NODEBUG_ALIAS = __lazy_tuple<decay_t<_Ts>...>;
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_LAZY
|
||||
@@ -1,655 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_LET_VALUE
|
||||
#define __CUDAX_EXECUTION_LET_VALUE
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__cccl/unreachable.h>
|
||||
#include <cuda/std/__exception/exception_macros.h>
|
||||
#include <cuda/std/__type_traits/common_type.h>
|
||||
#include <cuda/std/__type_traits/decay.h>
|
||||
#include <cuda/std/__type_traits/fold.h>
|
||||
#include <cuda/std/__type_traits/is_callable.h>
|
||||
#include <cuda/std/__utility/auto_cast.h>
|
||||
#include <cuda/std/__utility/pod_tuple.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/concepts.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/exception.cuh>
|
||||
#include <cuda/experimental/__execution/rcvr_ref.cuh>
|
||||
#include <cuda/experimental/__execution/rcvr_with_env.cuh>
|
||||
#include <cuda/experimental/__execution/transform_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/transform_sender.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__execution/variant.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __let_t
|
||||
{
|
||||
template <class _LetTag>
|
||||
static ::cuda::std::__undefined<_LetTag> __set_tag;
|
||||
|
||||
template <class _LetTag>
|
||||
using __set_tag_for_t = decltype(_LIBCUDACXX_AUTO_CAST(__set_tag<_LetTag>));
|
||||
|
||||
//! @brief Computes the type of a variant of tuples to hold the results of the
|
||||
//! predecessor sender.
|
||||
template <class _SetTag, class _Completions, class _Env>
|
||||
using __sndr1_results_t _CCCL_NODEBUG_ALIAS =
|
||||
__gather_completion_signatures<_Completions, _SetTag, ::cuda::std::__decayed_tuple, __variant>;
|
||||
|
||||
// This environment is part of the receiver used to connect the secondary sender.
|
||||
template <class _SetTag, class _Attrs, class... _Env>
|
||||
_CCCL_HOST_DEVICE_API static constexpr auto __mk_env2(const _Attrs& __attrs, const _Env&... __env) noexcept
|
||||
{
|
||||
if constexpr (__callable<get_completion_scheduler_t<_SetTag>, const _Attrs&, __fwd_env_t<const _Env&>...>)
|
||||
{
|
||||
return __mk_sch_env(get_completion_scheduler<_SetTag>(__attrs, __fwd_env(__env)...), __fwd_env(__env)...);
|
||||
}
|
||||
else if constexpr (__callable<get_completion_domain_t<_SetTag>, const _Attrs&, __fwd_env_t<const _Env&>...>)
|
||||
{
|
||||
using __domain_t = __call_result_t<get_completion_domain_t<_SetTag>, const _Attrs&, __fwd_env_t<const _Env&>...>;
|
||||
return prop{get_domain, __domain_t{}};
|
||||
}
|
||||
else
|
||||
{
|
||||
return env{};
|
||||
}
|
||||
}
|
||||
|
||||
template <class _SetTag, class _Attrs, class... _Env>
|
||||
using __env2_t _CCCL_NODEBUG_ALIAS =
|
||||
decltype(__let_t::__mk_env2<_SetTag>(::cuda::std::declval<_Attrs>(), ::cuda::std::declval<_Env>()...));
|
||||
|
||||
template <class _SetTag, class _Attrs, class _Env>
|
||||
using __join_env2_t _CCCL_NODEBUG_ALIAS = __join_env_t<__env2_t<_SetTag, _Attrs, _Env>, _Env>;
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr2_fn
|
||||
{
|
||||
template <class... _As>
|
||||
using __call _CCCL_NODEBUG_ALIAS = __call_result_t<_Fn, decay_t<_As>&...>;
|
||||
};
|
||||
|
||||
template <class _Rcvr, class _Env2>
|
||||
struct __sndr2_rcvr_t : __rcvr_ref_t<__rcvr_with_env_t<_Rcvr, _Env2>>
|
||||
{
|
||||
using __base_t = __rcvr_ref_t<__rcvr_with_env_t<_Rcvr, _Env2>>;
|
||||
|
||||
_CCCL_HOST_DEVICE_API explicit constexpr __sndr2_rcvr_t(__rcvr_with_env_t<_Rcvr, _Env2>& __rcvr) noexcept
|
||||
: __base_t(__ref_rcvr(__rcvr))
|
||||
{}
|
||||
};
|
||||
|
||||
template <class _Fn, class _Rcvr, class _Env2>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __state_base_t
|
||||
{
|
||||
//! @brief For a given set of result datums, compute the type of the secondary
|
||||
//! sender's operation state.
|
||||
template <class... _As>
|
||||
using __sndr2_opstate_fn _CCCL_NODEBUG_ALIAS =
|
||||
connect_result_t<::cuda::std::__type_call<__sndr2_fn<_Fn>, _As...>, __sndr2_rcvr_t<_Rcvr, _Env2>>;
|
||||
|
||||
__rcvr_with_env_t<_Rcvr, _Env2> __rcvr_;
|
||||
_Fn __fn_;
|
||||
};
|
||||
|
||||
template <class _SetTag, class _Fn, class _Rcvr, class _Env2, class _Completions>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __state_t : __state_base_t<_Fn, _Rcvr, _Env2>
|
||||
{
|
||||
using __sndr2_opstate_t _CCCL_NODEBUG_ALIAS =
|
||||
__gather_completion_signatures<_Completions,
|
||||
_SetTag,
|
||||
__state_t::__state_base_t::template __sndr2_opstate_fn,
|
||||
__variant>;
|
||||
|
||||
__sndr1_results_t<_SetTag, _Completions, __fwd_env_t<env_of_t<_Rcvr>>> __result_{};
|
||||
__sndr2_opstate_t __opstate2_{};
|
||||
};
|
||||
|
||||
//! @brief This is the receiver that gets connected to the predecessor sender. It caches
|
||||
//! the results of the predecessor and then calls the user-provided function with them
|
||||
//! to produce the secondary sender, which it then connects and starts.
|
||||
template <class _SetTag, class _Fn, class _Rcvr, class _Env2, class _Completions>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr1_rcvr_t
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
template <class... _As>
|
||||
_CCCL_HOST_DEVICE_API void __complete(_SetTag, _As&&... __as) noexcept
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
// Store the results so the lvalue refs we pass to the function will be valid for
|
||||
// the duration of the async op.
|
||||
auto& __tupl =
|
||||
__state_->__result_.template __emplace<::cuda::std::__decayed_tuple<_As...>>(static_cast<_As&&>(__as)...);
|
||||
|
||||
// Call the function with the results and connect the resulting sender, storing
|
||||
// the operation state in __state_->__opstate2_.
|
||||
auto& __next_op = __state_->__opstate2_.__emplace_from(
|
||||
execution::connect,
|
||||
::cuda::std::__apply(static_cast<_Fn&&>(__state_->__fn_), __tupl),
|
||||
__sndr2_rcvr_t(__state_->__rcvr_));
|
||||
execution::start(__next_op);
|
||||
}
|
||||
_CCCL_CATCH_ALL
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(__state_->__rcvr_.__base()), execution::current_exception());
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Tag, class... _As>
|
||||
_CCCL_HOST_DEVICE_API void __complete(_Tag, _As&&... __as) noexcept
|
||||
{
|
||||
// Forward the completion to the receiver unchanged.
|
||||
_Tag{}(static_cast<_Rcvr&&>(__state_->__rcvr_.__base()), static_cast<_As&&>(__as)...);
|
||||
}
|
||||
|
||||
template <class... _As>
|
||||
_CCCL_HOST_DEVICE_API void set_value(_As&&... __as) noexcept
|
||||
{
|
||||
__complete(execution::set_value, static_cast<_As&&>(__as)...);
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API void set_error(_Error&& __error) noexcept
|
||||
{
|
||||
__complete(execution::set_error, static_cast<_Error&&>(__error));
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void set_stopped() noexcept
|
||||
{
|
||||
__complete(execution::set_stopped);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __fwd_env_t<env_of_t<_Rcvr>>
|
||||
{
|
||||
return __fwd_env(execution::get_env(__state_->__rcvr_.__base()));
|
||||
}
|
||||
|
||||
__state_t<_SetTag, _Fn, _Rcvr, _Env2, _Completions>* __state_;
|
||||
};
|
||||
|
||||
//! @brief The `let_(value|error|stopped)` operation state.
|
||||
//! @tparam _CvSndr The cvref-qualified predecessor sender type.
|
||||
//! @tparam _Fn The user-provided function to be called with the result datums of the
|
||||
//! predecessor sender.
|
||||
//! @tparam _Rcvr The receiver connected to the `let_(value|error|stopped)`
|
||||
//! sender.
|
||||
template <class _SetTag, class _CvSndr, class _Fn, class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
using __completions_t = completion_signatures_of_t<_CvSndr, __fwd_env_t<env_of_t<_Rcvr>>>;
|
||||
using __env2_t = __let_t::__env2_t<_SetTag, env_of_t<_CvSndr>, env_of_t<_Rcvr>>;
|
||||
using __sndr1_rcvr_t = __sndr1_rcvr_t<_SetTag, _Fn, _Rcvr, __env2_t, __completions_t>;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __opstate_t(
|
||||
_CvSndr& __sndr,
|
||||
_Fn& __fn,
|
||||
_Rcvr& __rcvr,
|
||||
__env2_t&& __env2) noexcept(__nothrow_decay_copyable<_Fn, _Rcvr, __env2_t>
|
||||
&& __nothrow_connectable<_CvSndr, __sndr1_rcvr_t>)
|
||||
: __state_{{{static_cast<_Rcvr&&>(__rcvr), static_cast<__env2_t&&>(__env2)}, static_cast<_Fn&&>(__fn)}}
|
||||
, __opstate1_(execution::connect(static_cast<_CvSndr&&>(__sndr), __sndr1_rcvr_t{&__state_}))
|
||||
{}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __opstate_t(_CvSndr&& __sndr, _Fn __fn, _Rcvr __rcvr) noexcept(
|
||||
__nothrow_decay_copyable<_Fn, _Rcvr, __env2_t> && __nothrow_connectable<_CvSndr, __sndr1_rcvr_t>)
|
||||
: __opstate_t(
|
||||
__sndr, __fn, __rcvr, __let_t::__mk_env2<_SetTag>(execution::get_env(__sndr), execution::get_env(__rcvr)))
|
||||
{}
|
||||
|
||||
_CCCL_IMMOVABLE(__opstate_t);
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void start() noexcept
|
||||
{
|
||||
execution::start(__opstate1_);
|
||||
}
|
||||
|
||||
__state_t<_SetTag, _Fn, _Rcvr, __env2_t, __completions_t> __state_;
|
||||
connect_result_t<_CvSndr, __sndr1_rcvr_t> __opstate1_;
|
||||
};
|
||||
|
||||
template <class _SetTag, class _Fn, class _Attrs, class... _Env>
|
||||
struct __domain_transform_fn
|
||||
{
|
||||
template <class... _As>
|
||||
using __call _CCCL_NODEBUG_ALIAS =
|
||||
__compl_domain_t<_SetTag,
|
||||
::cuda::std::__type_call<__sndr2_fn<_Fn>, _As...>,
|
||||
__join_env2_t<_SetTag, _Attrs, _Env>...>;
|
||||
};
|
||||
|
||||
//! @tparam _SetTag The completion signal of the predecessor sender that triggers the
|
||||
//! function call. For let_value, this is `set_value`.
|
||||
//! @tparam _SetTag2 The completion signal of the let_ sender itself that is being
|
||||
//! queried. For example, you may be querying a let_value sender for its set_error
|
||||
//! completion domain.
|
||||
template <class _SetTag, class _SetTag2, class _Sndr, class _Fn, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto __get_completion_domain() noexcept
|
||||
{
|
||||
if constexpr (sender_in<_Sndr, _Env...>)
|
||||
{
|
||||
using __domain_transform_fn = __let_t::__domain_transform_fn<_SetTag2, _Fn, env_of_t<_Sndr>, _Env...>;
|
||||
return __gather_completion_signatures<completion_signatures_of_t<_Sndr, _Env...>,
|
||||
_SetTag,
|
||||
__domain_transform_fn::template __call,
|
||||
__common_domain_t>();
|
||||
}
|
||||
else
|
||||
{
|
||||
return __not_a_domain{};
|
||||
}
|
||||
}
|
||||
|
||||
template <class _SetTag, class _SetTag2, class _Sndr, class _Fn, class... _Env>
|
||||
using __let_completion_domain_t _CCCL_NODEBUG_ALIAS =
|
||||
__unless_one_of_t<decltype(__let_t::__get_completion_domain<_SetTag, _SetTag2, _Sndr, _Fn, _Env...>()),
|
||||
__not_a_domain>;
|
||||
|
||||
template <class _LetTag, class _Fn, class... _JoinEnv2>
|
||||
struct __transform_args_fn
|
||||
{
|
||||
template <class... _Ts>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto operator()() const
|
||||
{
|
||||
if constexpr (!__decay_copyable<_Ts...>)
|
||||
{
|
||||
return invalid_completion_signature<_WHERE(_IN_ALGORITHM, _LetTag),
|
||||
_WHAT(_ARGUMENTS_ARE_NOT_DECAY_COPYABLE),
|
||||
_WITH_ARGUMENTS(_Ts...)>();
|
||||
}
|
||||
else if constexpr (!::cuda::std::__type_callable<__sndr2_fn<_Fn>, _Ts...>::value)
|
||||
{
|
||||
return invalid_completion_signature<_WHERE(_IN_ALGORITHM, _LetTag),
|
||||
_WHAT(_FUNCTION_IS_NOT_CALLABLE),
|
||||
_WITH_FUNCTION(_Fn),
|
||||
_WITH_ARGUMENTS(decay_t<_Ts> & ...)>();
|
||||
}
|
||||
else
|
||||
{
|
||||
using __sndr2_t = ::cuda::std::__type_call<__sndr2_fn<_Fn>, _Ts...>;
|
||||
if constexpr (!sender<__sndr2_t>)
|
||||
{
|
||||
return invalid_completion_signature<_WHERE(_IN_ALGORITHM, _LetTag),
|
||||
_WHAT(_FUNCTION_MUST_RETURN_A_SENDER),
|
||||
_WITH_FUNCTION(_Fn),
|
||||
_WITH_ARGUMENTS(decay_t<_Ts> & ...),
|
||||
_WITH_RETURN_TYPE(__sndr2_t)>();
|
||||
}
|
||||
else
|
||||
{
|
||||
return get_completion_signatures<__sndr2_t, _JoinEnv2...>();
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Fn, class... _Env>
|
||||
struct __completion_behavior_transform_fn
|
||||
{
|
||||
template <class _Tag, class... _Ts>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Tag (*)(_Ts...)) const noexcept
|
||||
{
|
||||
if constexpr (::cuda::std::__type_callable<__sndr2_fn<_Fn>, _Ts...>::value)
|
||||
{
|
||||
using __sndr2_t = ::cuda::std::__type_call<__sndr2_fn<_Fn>, _Ts...>;
|
||||
return execution::get_completion_behavior<__sndr2_t, _Env...>();
|
||||
}
|
||||
else
|
||||
{
|
||||
return completion_behavior::unknown;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// A metafunction to check whether the predecessor's completion results are nothrow
|
||||
// decay-copyable and whether connecting the secondary sender is nothrow.
|
||||
template <class _SetTag, class _Sndr, class _Fn, class _Env>
|
||||
struct __has_nothrow_completions_fn
|
||||
{
|
||||
using __env2_t = __let_t::__join_env2_t<_SetTag, env_of_t<_Sndr>, _Env>;
|
||||
|
||||
template <class... _Ts>
|
||||
using __call _CCCL_NODEBUG_ALIAS = ::cuda::std::bool_constant<
|
||||
(noexcept(_LIBCUDACXX_AUTO_CAST(declval<_Ts>())) && ...)
|
||||
&& noexcept(execution::connect(declval<_Fn>()(declval<decay_t<_Ts>&>()...), __receiver_archetype<__env2_t>()))>;
|
||||
};
|
||||
|
||||
template <class _SetTag, class _Sndr, class _Fn, class _Env>
|
||||
using __has_nothrow_completions _CCCL_NODEBUG_ALIAS =
|
||||
__gather_completion_signatures<completion_signatures_of_t<_Sndr, _Env>,
|
||||
_SetTag,
|
||||
__has_nothrow_completions_fn<_SetTag, _Sndr, _Fn, _Env>::template __call,
|
||||
::cuda::std::__type_strict_and::__call>;
|
||||
|
||||
template <class _LetTag, class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __attrs_t;
|
||||
|
||||
template <class _LetTag, class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
template <class _LetTag, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_HIDDEN __closure_t // hidden visibility because member __fn_ is hidden if it is an
|
||||
// extended (host/device) lambda
|
||||
{
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto operator()(_Sndr __sndr) &&
|
||||
{
|
||||
return _LetTag()(static_cast<_Sndr&&>(__sndr), static_cast<_Fn&&>(__fn_));
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto operator()(_Sndr __sndr) const&
|
||||
{
|
||||
return _LetTag()(static_cast<_Sndr&&>(__sndr), __fn_);
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend auto operator|(_Sndr __sndr, __closure_t __self)
|
||||
{
|
||||
return _LetTag()(static_cast<_Sndr&&>(__sndr), static_cast<_Fn&&>(__self.__fn_));
|
||||
}
|
||||
|
||||
_Fn __fn_;
|
||||
};
|
||||
};
|
||||
|
||||
template <class _LetTag>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __let_base_t : __let_t
|
||||
{
|
||||
//! @brief The `let_(value|error|stopped)` sender.
|
||||
//! @tparam _Sndr The predecessor sender.
|
||||
//! @tparam _Fn The function to be called when the predecessor sender
|
||||
//! completes.
|
||||
template <class _Sndr, class _Fn>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr __sndr, _Fn __fn) const;
|
||||
|
||||
template <class _Fn>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Fn __fn) const noexcept;
|
||||
};
|
||||
|
||||
struct let_value_t : __let_base_t<let_value_t>
|
||||
{
|
||||
template <class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t;
|
||||
};
|
||||
|
||||
struct let_error_t : __let_base_t<let_error_t>
|
||||
{
|
||||
template <class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t;
|
||||
};
|
||||
|
||||
struct let_stopped_t : __let_base_t<let_stopped_t>
|
||||
{
|
||||
template <class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t;
|
||||
};
|
||||
|
||||
template <class _LetTag, class _Sndr, class _Fn>
|
||||
struct __let_t::__attrs_t
|
||||
{
|
||||
using __set_tag_t = __set_tag_for_t<_LetTag>;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __attrs_t(const __sndr_t<_LetTag, _Sndr, _Fn>& __self) noexcept
|
||||
: __self_(__self)
|
||||
{}
|
||||
|
||||
template <class _Tag>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto query(get_completion_scheduler_t<_Tag>) const = delete;
|
||||
|
||||
template <class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_domain_t<__set_tag_t>, const _Env&...) const noexcept
|
||||
-> __let_completion_domain_t<__set_tag_t, __set_tag_t, _Sndr, _Fn, _Env...>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Tag, class... _Env)
|
||||
_CCCL_REQUIRES(::cuda::std::__is_included_in_v<_Tag, set_error_t, set_stopped_t> _CCCL_AND(
|
||||
__has_nothrow_completions<__set_tag_t, _Sndr, _Fn, _Env>::value&&...))
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_domain_t<_Tag>, const _Env&...) const noexcept
|
||||
-> __common_domain_t<__compl_domain_t<_Tag, _Sndr, __fwd_env_t<_Env>...>,
|
||||
__let_completion_domain_t<__set_tag_t, _Tag, _Sndr, _Fn, _Env...>>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Env)
|
||||
_CCCL_REQUIRES((!__has_nothrow_completions<__set_tag_t, _Sndr, _Fn, _Env>::value))
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_domain_t<set_error_t>, const _Env&) const noexcept
|
||||
-> __common_domain_t<__compl_domain_t<__set_tag_t, _Sndr, __fwd_env_t<_Env>>,
|
||||
__compl_domain_t<set_error_t, _Sndr, __fwd_env_t<_Env>>,
|
||||
__let_completion_domain_t<__set_tag_t, set_error_t, _Sndr, _Fn, _Env>>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
template <class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_behavior_t, const _Env&...) const noexcept
|
||||
{
|
||||
#if _CCCL_CUDACC_BELOW(12, 9)
|
||||
if constexpr (sender_in<_Sndr> || (sender_in<_Sndr, __fwd_env_t<_Env>> && ...))
|
||||
#else
|
||||
if constexpr (sender_in<_Sndr, __fwd_env_t<_Env>...>)
|
||||
#endif
|
||||
{
|
||||
// The completion behavior of let_value(sndr, fn) is the weakest completion
|
||||
// behavior of sndr and all the senders that fn can potentially produce. (MSVC
|
||||
// needs the constexpr computation broken up, hence the local variables.)
|
||||
constexpr auto __completions =
|
||||
execution::get_completion_signatures<_Sndr, __fwd_env_t<_Env>...>().select(__set_tag_t{});
|
||||
constexpr auto __transform_fn =
|
||||
__completion_behavior_transform_fn<_Fn, __join_env2_t<__set_tag_t, env_of_t<_Sndr>, _Env>...>{};
|
||||
constexpr auto __behavior = __completions.transform_reduce(__transform_fn, execution::min);
|
||||
return (execution::min) (execution::get_completion_behavior<_Sndr, __fwd_env_t<_Env>...>(), __behavior);
|
||||
}
|
||||
else
|
||||
{
|
||||
return completion_behavior::unknown;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const __sndr_t<_LetTag, _Sndr, _Fn>& __self_;
|
||||
};
|
||||
|
||||
template <class _LetTag, class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __let_t::__sndr_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
using __set_tag_t = __set_tag_for_t<_LetTag>;
|
||||
|
||||
template <class _Self, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto __get_completion_signatures()
|
||||
{
|
||||
_CUDAX_LET_COMPLETIONS(auto(__child_completions) = get_child_completion_signatures<_Self, _Sndr, _Env...>())
|
||||
{
|
||||
constexpr auto __transform_fn =
|
||||
__transform_args_fn<_LetTag, _Fn, __join_env2_t<__set_tag_t, env_of_t<_Sndr>, _Env>...>{};
|
||||
|
||||
if constexpr (__set_tag_t{} == execution::set_value)
|
||||
{
|
||||
return transform_completion_signatures(__child_completions, __transform_fn);
|
||||
}
|
||||
else if constexpr (__set_tag_t{} == execution::set_error)
|
||||
{
|
||||
return transform_completion_signatures(__child_completions, {}, __transform_fn);
|
||||
}
|
||||
else
|
||||
{
|
||||
return transform_completion_signatures(__child_completions, {}, {}, __transform_fn);
|
||||
}
|
||||
}
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
template <class _Self, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
_CUDAX_LET_COMPLETIONS(auto(__completions) = __get_completion_signatures<_Self, _Env...>())
|
||||
{
|
||||
// If we do not have an environment yet, assume that connecting the secondary sender
|
||||
// might throw.
|
||||
constexpr bool __nothrow_completions = (__has_nothrow_completions<__set_tag_t, _Sndr, _Fn, _Env>::value || ...);
|
||||
constexpr auto __eptr_completion = execution::__eptr_completion_if<!__nothrow_completions>();
|
||||
return __completions + __eptr_completion;
|
||||
}
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
_CCCL_HOST_DEVICE_API auto connect(_Rcvr __rcvr) && noexcept(
|
||||
__nothrow_constructible<__opstate_t<__set_tag_t, _Sndr, _Fn, _Rcvr>, _Sndr, _Fn, _Rcvr>)
|
||||
-> __opstate_t<__set_tag_t, _Sndr, _Fn, _Rcvr>
|
||||
{
|
||||
return __opstate_t<__set_tag_t, _Sndr, _Fn, _Rcvr>(
|
||||
static_cast<_Sndr&&>(__sndr_), static_cast<_Fn&&>(__fn_), static_cast<_Rcvr&&>(__rcvr));
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) const& noexcept(
|
||||
__nothrow_constructible<__opstate_t<__set_tag_t, const _Sndr&, _Fn, _Rcvr>, const _Sndr&, const _Fn&, _Rcvr>)
|
||||
-> __opstate_t<__set_tag_t, const _Sndr&, _Fn, _Rcvr>
|
||||
{
|
||||
return __opstate_t<__set_tag_t, const _Sndr&, _Fn, _Rcvr>(__sndr_, __fn_, static_cast<_Rcvr&&>(__rcvr));
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept
|
||||
{
|
||||
return __attrs_t<_LetTag, _Sndr, _Fn>(*this);
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ _LetTag __tag_;
|
||||
_Fn __fn_;
|
||||
_Sndr __sndr_;
|
||||
};
|
||||
|
||||
template <class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT let_value_t::__sndr_t : __let_t::__sndr_t<let_value_t, _Sndr, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT let_error_t::__sndr_t : __let_t::__sndr_t<let_error_t, _Sndr, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT let_stopped_t::__sndr_t : __let_t::__sndr_t<let_stopped_t, _Sndr, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT let_value_t::__closure_t : __let_t::__closure_t<let_value_t, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT let_error_t::__closure_t : __let_t::__closure_t<let_error_t, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT let_stopped_t::__closure_t : __let_t::__closure_t<let_stopped_t, _Fn>
|
||||
{};
|
||||
|
||||
template <class... _Sndr>
|
||||
using __all_non_dependent_t = ::cuda::std::__fold_and<(!dependent_sender<_Sndr>) ...>;
|
||||
|
||||
template <class _LetTag>
|
||||
template <class _Sndr, class _Fn>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __let_base_t<_LetTag>::operator()(_Sndr __sndr, _Fn __fn) const
|
||||
{
|
||||
using __sndr_t = typename _LetTag::template __sndr_t<_Sndr, _Fn>;
|
||||
|
||||
// If the incoming sender is non-dependent, we can check the completion signatures of
|
||||
// the composed sender immediately.
|
||||
if constexpr (!dependent_sender<_Sndr>)
|
||||
{
|
||||
// Although the input sender is not dependent, the sender(s) returned from the
|
||||
// function might be. Only do eager type-checking if all the possible senders returned
|
||||
// by the function are non-dependent. If any of them is dependent, we will defer the
|
||||
// type-checking to the point where the sender is connected.
|
||||
using __completions_t = completion_signatures_of_t<_Sndr>;
|
||||
constexpr bool __all_non_dependent =
|
||||
__gather_completion_signatures<__completions_t,
|
||||
__set_tag_for_t<_LetTag>,
|
||||
__sndr2_fn<_Fn>::template __call,
|
||||
__all_non_dependent_t>::value;
|
||||
|
||||
if constexpr (__all_non_dependent)
|
||||
{
|
||||
execution::__assert_valid_completion_signatures(get_completion_signatures<__sndr_t>());
|
||||
}
|
||||
}
|
||||
|
||||
return __sndr_t{{{}, static_cast<_Fn&&>(__fn), static_cast<_Sndr&&>(__sndr)}};
|
||||
}
|
||||
|
||||
template <class _LetTag>
|
||||
template <class _Fn>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __let_base_t<_LetTag>::operator()(_Fn __fn) const noexcept
|
||||
{
|
||||
using __closure_t = typename _LetTag::template __closure_t<_Fn>;
|
||||
return __closure_t{{static_cast<_Fn&&>(__fn)}};
|
||||
}
|
||||
|
||||
template <>
|
||||
constexpr set_value_t __let_t::__set_tag<let_value_t>{};
|
||||
template <>
|
||||
constexpr set_error_t __let_t::__set_tag<let_error_t>{};
|
||||
template <>
|
||||
constexpr set_stopped_t __let_t::__set_tag<let_stopped_t>{};
|
||||
|
||||
template <class _Sndr, class _Fn>
|
||||
inline constexpr int structured_binding_size<let_value_t::__sndr_t<_Sndr, _Fn>> = 3;
|
||||
template <class _Sndr, class _Fn>
|
||||
inline constexpr int structured_binding_size<let_error_t::__sndr_t<_Sndr, _Fn>> = 3;
|
||||
template <class _Sndr, class _Fn>
|
||||
inline constexpr int structured_binding_size<let_stopped_t::__sndr_t<_Sndr, _Fn>> = 3;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto let_value = let_value_t{};
|
||||
_CCCL_GLOBAL_CONSTANT auto let_error = let_error_t{};
|
||||
_CCCL_GLOBAL_CONSTANT auto let_stopped = let_stopped_t{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_LET_VALUE
|
||||
@@ -1,170 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_META
|
||||
#define __CUDAX_EXECUTION_META
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__type_traits/conditional.h>
|
||||
#include <cuda/std/__type_traits/integral_constant.h>
|
||||
#include <cuda/std/__type_traits/is_valid_expansion.h>
|
||||
#include <cuda/std/__type_traits/type_list.h>
|
||||
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/diagnostics.cuh>
|
||||
|
||||
#if __cpp_lib_three_way_comparison >= 201907L
|
||||
# include <compare> // IWYU pragma: keep
|
||||
#endif // __cpp_lib_three_way_comparison >= 201907L
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
template <bool>
|
||||
struct __type_try__;
|
||||
|
||||
template <>
|
||||
struct __type_try__<false>
|
||||
{
|
||||
template <template <class...> class _Fn, class... _Ts>
|
||||
using __call_q _CCCL_NODEBUG_ALIAS = _Fn<_Ts...>;
|
||||
|
||||
template <class _Fn, class... _Ts>
|
||||
using __call _CCCL_NODEBUG_ALIAS = typename _Fn::template __call<_Ts...>;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct __type_try__<true>
|
||||
{
|
||||
template <template <class...> class _Fn, class... _Ts>
|
||||
using __call_q _CCCL_NODEBUG_ALIAS = __type_find_error<_Ts...>;
|
||||
|
||||
template <class _Fn, class... _Ts>
|
||||
using __call _CCCL_NODEBUG_ALIAS = __type_find_error<_Fn, _Ts...>;
|
||||
};
|
||||
|
||||
template <class _Fn, class... _Ts>
|
||||
using __type_try_call _CCCL_NODEBUG_ALIAS =
|
||||
typename __type_try__<__type_contains_error<_Fn, _Ts...>>::template __call<_Fn, _Ts...>;
|
||||
|
||||
template <template <class...> class _Fn, class... _Ts>
|
||||
using __type_try_call_quote _CCCL_NODEBUG_ALIAS =
|
||||
typename __type_try__<__type_contains_error<_Ts...>>::template __call_q<_Fn, _Ts...>;
|
||||
|
||||
// wraps a meta-callable such that if any of the arguments are errors, the
|
||||
// result is an error.
|
||||
template <class _Fn>
|
||||
struct __type_try
|
||||
{
|
||||
template <class... _Ts>
|
||||
using __call _CCCL_NODEBUG_ALIAS = __type_try_call<_Fn, _Ts...>;
|
||||
};
|
||||
|
||||
template <template <class...> class _Fn, class... _Default>
|
||||
struct __type_try_quote;
|
||||
|
||||
// equivalent to __type_try<__type_quote<_Fn>>
|
||||
template <template <class...> class _Fn>
|
||||
struct __type_try_quote<_Fn>
|
||||
{
|
||||
template <class... _Ts>
|
||||
using __call _CCCL_NODEBUG_ALIAS =
|
||||
typename __type_try__<__type_contains_error<_Ts...>>::template __call_q<_Fn, _Ts...>;
|
||||
};
|
||||
|
||||
// equivalent to __type_try<__type_quote<_Fn, _Default>>
|
||||
template <template <class...> class _Fn, class _Default>
|
||||
struct __type_try_quote<_Fn, _Default>
|
||||
{
|
||||
template <class... _Ts>
|
||||
using __call _CCCL_NODEBUG_ALIAS =
|
||||
typename ::cuda::std::_If<__is_instantiable_with<_Fn, _Ts...>, //
|
||||
__type_try_quote<_Fn>,
|
||||
::cuda::std::__type_always<_Default>>::template __call<_Ts...>;
|
||||
};
|
||||
|
||||
template <class _Return>
|
||||
struct __type_function
|
||||
{
|
||||
template <class... _Args>
|
||||
using __call _CCCL_NODEBUG_ALIAS = _Return(_Args...);
|
||||
};
|
||||
|
||||
template <class _Return>
|
||||
struct __type_function1
|
||||
{
|
||||
template <class _Arg>
|
||||
using __call _CCCL_NODEBUG_ALIAS = _Return(_Arg);
|
||||
};
|
||||
|
||||
template <class _First, class _Second>
|
||||
using __type_first = _First;
|
||||
|
||||
template <class _First, class _Second>
|
||||
using __type_second = _Second;
|
||||
|
||||
template <template <class...> class _Second, template <class...> class _First>
|
||||
struct __type_compose_quote
|
||||
{
|
||||
template <class... _Ts>
|
||||
using __call _CCCL_NODEBUG_ALIAS = _Second<_First<_Ts...>>;
|
||||
};
|
||||
|
||||
struct __type_count
|
||||
{
|
||||
template <class... _Ts>
|
||||
using __call _CCCL_NODEBUG_ALIAS = ::cuda::std::integral_constant<size_t, sizeof...(_Ts)>;
|
||||
};
|
||||
|
||||
template <class _Continuation>
|
||||
struct __type_concat_into
|
||||
{
|
||||
template <class... _Args>
|
||||
using __call _CCCL_NODEBUG_ALIAS =
|
||||
::cuda::std::__type_call1<::cuda::std::__type_concat<::cuda::std::__as_type_list<_Args>...>, _Continuation>;
|
||||
};
|
||||
|
||||
template <template <class...> class _Continuation>
|
||||
struct __type_concat_into_quote : __type_concat_into<::cuda::std::__type_quote<_Continuation>>
|
||||
{};
|
||||
|
||||
template <class _Ty>
|
||||
struct __type_self_or
|
||||
{
|
||||
template <class _Uy = _Ty>
|
||||
using __call _CCCL_NODEBUG_ALIAS = _Uy;
|
||||
};
|
||||
|
||||
template <template <class...> class _Fn, class _Default, class... _Ts>
|
||||
using __type_call_or_quote =
|
||||
typename ::cuda::std::_If<__is_instantiable_with<_Fn, _Ts...>,
|
||||
::cuda::std::__type_quote<_Fn>,
|
||||
::cuda::std::__type_always<_Default>>::template __call<_Ts...>;
|
||||
|
||||
template <class _Fn, class _Default, class... _Ts>
|
||||
using __type_call_or =
|
||||
typename ::cuda::std::_If<__is_instantiable_with<_Fn::template __call, _Ts...>,
|
||||
_Fn,
|
||||
::cuda::std::__type_always<_Default>>::template __call<_Ts...>;
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_META
|
||||
@@ -1,299 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_ON
|
||||
#define __CUDAX_EXECUTION_ON
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/concepts.cuh>
|
||||
#include <cuda/experimental/__execution/continues_on.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/domain.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/meta.cuh>
|
||||
#include <cuda/experimental/__execution/queries.cuh>
|
||||
#include <cuda/experimental/__execution/sndr_ref.cuh>
|
||||
#include <cuda/experimental/__execution/starts_on.cuh>
|
||||
#include <cuda/experimental/__execution/transform_sender.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
#include <cuda/experimental/__execution/write_env.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
//! @brief Sender adaptor that transfers execution to a specified scheduler and back.
|
||||
//!
|
||||
//! The `on` algorithm provides execution context control by moving computation to
|
||||
//! different execution resources. It has two primary forms:
|
||||
//!
|
||||
//! ## Form 1: `on(scheduler, sender)`
|
||||
//!
|
||||
//! Starts a sender on an execution agent belonging to the specified scheduler's execution
|
||||
//! resource, and upon completion, transfers execution back to the original execution
|
||||
//! resource where the `on` sender was started.
|
||||
//!
|
||||
//! @code
|
||||
//! auto sndr = on(gpu_scheduler, some_computation);
|
||||
//! auto [result] = sync_wait(std::move(sndr)).value();
|
||||
//! @endcode
|
||||
//!
|
||||
//! ## Form 2: `on(sender, scheduler, closure)` or `sender | on(scheduler, closure)`
|
||||
//!
|
||||
//! Upon completion of the input sender, transfers execution to the specified scheduler's
|
||||
//! execution resource, executes the closure with the sender's results, and then transfers
|
||||
//! execution back to where the original sender completed.
|
||||
//!
|
||||
//! @code
|
||||
//! auto sndr = some_computation | on(gpu_scheduler, then([](auto value) { /*...*/ }));
|
||||
//! auto [result] = sync_wait(std::move(sndr)).value();
|
||||
//! @endcode
|
||||
//!
|
||||
//! ## Behavior
|
||||
//!
|
||||
//! - **Form 1**: Execution flow: current → target scheduler → back to current
|
||||
//! - **Form 2**: Execution flow: current → (sender completes) → target scheduler → back to sender's completion context
|
||||
//!
|
||||
//! The algorithm remembers the original scheduler context and ensures execution returns
|
||||
//! to it after the target scheduler's work is complete. If no scheduler is available in
|
||||
//! the current execution context, the operation is ill-formed and results in a
|
||||
//! compilation error.
|
||||
//!
|
||||
//! ## Error Handling
|
||||
//!
|
||||
//! If any scheduling operation fails, an error completion is executed on an unspecified
|
||||
//! execution agent.
|
||||
//!
|
||||
//! @note This is CUDA's experimental implementation of the C++26 `std::execution::on` algorithm
|
||||
//! as specified in [exec.on].
|
||||
//!
|
||||
//! @see @c starts_on and @c continues_on for related scheduling primitives
|
||||
struct on_t
|
||||
{
|
||||
template <class _Sch, class _Sndr, class... _Closure>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
_CUDAX_SEMI_PRIVATE :
|
||||
template <class _Sndr, class _NewSch, class _OldSch, class... _Closure>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __lowered_sndr_t;
|
||||
|
||||
struct __lower_sndr_fn
|
||||
{
|
||||
// This is the the lowering for the `on(sch, sndr)` case
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sndr, class _NewSch, class _OldSch>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
operator()(_Sndr __sndr, _NewSch __new_sch, _OldSch __old_sch) const
|
||||
{
|
||||
return continues_on(starts_on(static_cast<_NewSch&&>(__new_sch), static_cast<_Sndr&&>(__sndr)), __old_sch);
|
||||
}
|
||||
|
||||
// This is the the lowering for the `sndr | on(sch, clsr)` case
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sndr, class _NewSch, class _OldSch, class _Closure>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
operator()(_Sndr __sndr, _NewSch __new_sch, _OldSch __old_sch, _Closure&& __closure) const
|
||||
{
|
||||
return continues_on(static_cast<_Closure&&>(__closure)(continues_on(static_cast<_Sndr&&>(__sndr), __new_sch)),
|
||||
__old_sch);
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Sch, class _Closure>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr __sndr) &&
|
||||
{
|
||||
return on_t{}(static_cast<_Sndr&&>(__sndr), __sch_, static_cast<_Closure&&>(__closure_));
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr __sndr) const&
|
||||
{
|
||||
return on_t{}(static_cast<_Sndr&&>(__sndr), __sch_, __closure_);
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr auto operator|(_Sndr __sndr, __closure_t __self)
|
||||
{
|
||||
return on_t{}(static_cast<_Sndr&&>(__sndr), __self.__sch_, static_cast<_Closure&&>(__self.__closure_));
|
||||
}
|
||||
|
||||
_Sch __sch_;
|
||||
_Closure __closure_;
|
||||
};
|
||||
|
||||
template <class _Sch, class _Sndr, class... _Closure>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __attrs_t
|
||||
{
|
||||
template <class _Env>
|
||||
using __new_sndr_t =
|
||||
__call_result_t<__lower_sndr_fn, __sndr_ref<const _Sndr&>, _Sch, __scheduler_of_t<_Env>, const _Closure&...>;
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Tag, class _Env)
|
||||
_CCCL_REQUIRES(__queryable_with<env_of_t<__new_sndr_t<_Env>>, _Tag, _Env>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(_Tag, const _Env& __env) const
|
||||
noexcept(__nothrow_queryable_with<env_of_t<__new_sndr_t<_Env>>, _Tag, _Env>) -> decltype(auto)
|
||||
{
|
||||
if constexpr (sizeof...(_Closure) == 0)
|
||||
{
|
||||
auto __tmp_sndr = __lower_sndr_fn()(__sndr_ref(__self_->__sndr_), __self_->__sch_, get_scheduler(__env));
|
||||
return execution::get_env(__tmp_sndr).query(_Tag(), __env);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto& [__sch, __closure] = __self_->__sch_closure_;
|
||||
auto __tmp_sndr = __lower_sndr_fn()(__sndr_ref(__self_->__sndr_), __sch, get_scheduler(__env), __closure);
|
||||
return execution::get_env(__tmp_sndr).query(_Tag(), __env);
|
||||
}
|
||||
}
|
||||
|
||||
const __sndr_t<_Sch, _Sndr, _Closure...>* __self_;
|
||||
};
|
||||
|
||||
public:
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Sch, class _Sndr)
|
||||
_CCCL_REQUIRES(__is_scheduler<_Sch> _CCCL_AND __is_sender<_Sndr>)
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Sch __sch, _Sndr __sndr) const
|
||||
{
|
||||
static_assert(__is_scheduler<_Sch>);
|
||||
return __sndr_t<_Sch, _Sndr>{{}, __sch, static_cast<_Sndr&&>(__sndr)};
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Sch, class _Closure)
|
||||
_CCCL_REQUIRES(__is_scheduler<_Sch> _CCCL_AND(!__is_sender<_Closure>))
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Sch __sch, _Closure __closure) const
|
||||
{
|
||||
static_assert(__is_scheduler<_Sch>);
|
||||
return __closure_t<_Sch, _Closure>{__sch, static_cast<_Closure&&>(__closure)};
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sndr, class _Sch, class _Closure>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr __sndr, _Sch __sch, _Closure __closure) const
|
||||
{
|
||||
static_assert(__is_scheduler<_Sch>);
|
||||
static_assert(__is_sender<_Sndr>);
|
||||
using __sndr_t = on_t::__sndr_t<_Sch, _Sndr, _Closure>;
|
||||
return __sndr_t{{}, {__sch, static_cast<_Closure&&>(__closure)}, static_cast<_Sndr&&>(__sndr)};
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sndr, class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto
|
||||
transform_sender(set_value_t, _Sndr&& __sndr, const _Env& __env)
|
||||
{
|
||||
using __not_a_scheduler =
|
||||
execution::__not_a_scheduler<_WHAT(_THE_ENVIRONMENT_OF_THE_RECEIVER_DOES_NOT_HAVE_A_SCHEDULER_FOR_ON_TO_RETURN_TO),
|
||||
_WHERE(_IN_ALGORITHM, on_t),
|
||||
_WITH_ENVIRONMENT(_Env)>;
|
||||
|
||||
auto&& [__ign, __data, __child] = __sndr;
|
||||
if constexpr (__is_scheduler<decltype(__data)>)
|
||||
{
|
||||
// The on(sch, sndr) case:
|
||||
auto __old_sch = __call_or(get_scheduler, __not_a_scheduler{}, __env);
|
||||
using __sndr_t = __lowered_sndr_t<decltype(__child), decltype(__data), decltype(__old_sch)>;
|
||||
static_assert(sender_for<__sndr_t, continues_on_t>);
|
||||
return __sndr_t{::cuda::std::forward_like<_Sndr>(__child), __data, __old_sch};
|
||||
}
|
||||
else
|
||||
{
|
||||
// The on(sndr, sch, closure) case:
|
||||
auto& [__new_sch, __closure] = __data;
|
||||
auto __old_sch =
|
||||
__call_or(get_completion_scheduler<set_value_t>, __not_a_scheduler{}, execution::get_env(__child), __env);
|
||||
using __sndr_t =
|
||||
__lowered_sndr_t<decltype(__child), decltype(__new_sch), decltype(__old_sch), decltype(__closure)>;
|
||||
return __sndr_t{
|
||||
::cuda::std::forward_like<_Sndr>(__child), __new_sch, __old_sch, ::cuda::std::forward_like<_Sndr>(__closure)};
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Sndr, class _NewSch, class _OldSch, class... _Closure>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT on_t::__lowered_sndr_t
|
||||
: __call_result_t<on_t::__lower_sndr_fn, _Sndr, _NewSch, _OldSch, _Closure...>
|
||||
{
|
||||
using __base_t = __call_result_t<on_t::__lower_sndr_fn, _Sndr, _NewSch, _OldSch, _Closure...>;
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _CvrefSndr>
|
||||
_CCCL_HOST_DEVICE_API constexpr __lowered_sndr_t(
|
||||
_CvrefSndr&& __sndr, _NewSch __new_sch, _OldSch __old_sch, _Closure... __closure)
|
||||
: __base_t{on_t::__lower_sndr_fn{}(
|
||||
static_cast<_CvrefSndr&&>(__sndr), __new_sch, __old_sch, static_cast<_Closure&&>(__closure)...)}
|
||||
{}
|
||||
};
|
||||
|
||||
// This is the sender used for `on(sch, sndr)`
|
||||
template <class _Sch, class _Sndr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT on_t::__sndr_t<_Sch, _Sndr>
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __attrs_t<_Sch, _Sndr>
|
||||
{
|
||||
return __attrs_t<_Sch, _Sndr>{this};
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ on_t __tag_;
|
||||
_Sch __sch_;
|
||||
_Sndr __sndr_;
|
||||
};
|
||||
|
||||
// This is the sender used for `on(sndr, sch, closure)` and `sndr | on(sch, closure)`.
|
||||
template <class _Sch, class _Sndr, class _Closure>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT on_t::__sndr_t<_Sch, _Sndr, _Closure>
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __attrs_t<_Sch, _Sndr, _Closure>
|
||||
{
|
||||
return __attrs_t<_Sch, _Sndr, _Closure>{this};
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ on_t __tag_;
|
||||
__closure_t<_Sch, _Closure> __sch_closure_;
|
||||
_Sndr __sndr_;
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT on_t on{};
|
||||
|
||||
template <class _Sch, class _Sndr>
|
||||
inline constexpr int structured_binding_size<on_t::__sndr_t<_Sch, _Sndr>> = 3;
|
||||
|
||||
template <class _Sch, class _Sndr, class _Closure>
|
||||
inline constexpr int structured_binding_size<on_t::__sndr_t<_Sch, _Sndr, _Closure>> = 3;
|
||||
|
||||
template <class _Sndr, class _NewSch, class _OldSch, class... _Closure>
|
||||
inline constexpr int structured_binding_size<on_t::__lowered_sndr_t<_Sndr, _NewSch, _OldSch, _Closure...>> = 3;
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_ON
|
||||
@@ -1,213 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_PARALLEL_SCHEDULER
|
||||
#define __CUDAX_EXECUTION_PARALLEL_SCHEDULER
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__type_traits/is_specialization_of.h>
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__memory/allocator.h>
|
||||
#include <cuda/std/__memory/allocator_traits.h>
|
||||
#include <cuda/std/__type_traits/type_list.h>
|
||||
#include <cuda/std/__utility/typeid.h>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/optional>
|
||||
#include <cuda/std/span>
|
||||
|
||||
#include <cuda/experimental/__execution/any_allocator.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/stop_token.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(4702) // warning C4702: unreachable code
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
namespace __detail
|
||||
{
|
||||
struct __env_proxy : __immovable
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API virtual auto query(const get_stop_token_t&) const noexcept -> inplace_stop_token = 0;
|
||||
_CCCL_HOST_DEVICE_API virtual auto query(const get_allocator_t&) const noexcept
|
||||
-> any_allocator<::cuda::std::byte> = 0;
|
||||
_CCCL_HOST_DEVICE_API virtual auto query(const get_scheduler_t&) const noexcept -> task_scheduler = 0;
|
||||
};
|
||||
} // namespace __detail
|
||||
|
||||
class receiver_proxy : __detail::__env_proxy
|
||||
{
|
||||
public:
|
||||
_CCCL_HOST_DEVICE_API virtual ~receiver_proxy() = 0;
|
||||
|
||||
_CCCL_HOST_DEVICE_API virtual void set_value() noexcept = 0;
|
||||
_CCCL_HOST_DEVICE_API virtual void set_error(exception_ptr&&) noexcept = 0;
|
||||
_CCCL_HOST_DEVICE_API virtual void set_stopped() noexcept = 0;
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API auto get_env() const noexcept -> const __detail::__env_proxy&
|
||||
{
|
||||
return *this;
|
||||
}
|
||||
|
||||
// _CCCL_EXEC_CHECK_DISABLE
|
||||
// _CCCL_TEMPLATE(class _Value, class Query)
|
||||
// _CCCL_REQUIRES(__callable<const __detail::__try_queryable&, Query, ::cuda::std::optional<_Value>&>)
|
||||
// [[nodiscard]] _CCCL_HOST_DEVICE_API auto try_query(const Query& __query) const noexcept ->
|
||||
// ::cuda::std::optional<_Value>
|
||||
// {
|
||||
// const __detail::__try_queryable& __queryable = *this;
|
||||
// ::cuda::std::optional<_Value> __value;
|
||||
// __queryable(__query, __value);
|
||||
// return __value;
|
||||
// }
|
||||
};
|
||||
|
||||
inline receiver_proxy::~receiver_proxy() = default;
|
||||
|
||||
struct bulk_item_receiver_proxy : receiver_proxy
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API virtual void execute(size_t, size_t) noexcept = 0;
|
||||
};
|
||||
|
||||
struct parallel_scheduler_backend
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API virtual ~parallel_scheduler_backend() = 0;
|
||||
|
||||
_CCCL_HOST_DEVICE_API virtual void schedule(receiver_proxy&, ::cuda::std::span<::cuda::std::byte>) noexcept = 0;
|
||||
|
||||
_CCCL_HOST_DEVICE_API virtual void
|
||||
schedule_bulk_chunked(size_t, bulk_item_receiver_proxy&, ::cuda::std::span<::cuda::std::byte>) noexcept = 0;
|
||||
|
||||
_CCCL_HOST_DEVICE_API virtual void
|
||||
schedule_bulk_unchunked(size_t, bulk_item_receiver_proxy&, ::cuda::std::span<::cuda::std::byte>) noexcept = 0;
|
||||
};
|
||||
|
||||
inline parallel_scheduler_backend::~parallel_scheduler_backend() = default;
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
// Partially implements the _RcvrProxy interface (either receiver_proxy or
|
||||
// bulk_item_receiver_proxy) in terms of a concrete receiver type _Rcvr.
|
||||
template <class _Rcvr, class _RcvrProxy>
|
||||
struct __receiver_proxy_base : _RcvrProxy
|
||||
{
|
||||
public:
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
_CCCL_HOST_DEVICE_API explicit __receiver_proxy_base(_Rcvr rcvr) noexcept
|
||||
: __rcvr_(static_cast<_Rcvr&&>(rcvr))
|
||||
{}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void set_error(exception_ptr&& eptr) noexcept final override
|
||||
{
|
||||
execution::set_error(_CCCL_MOVE(__rcvr_), _CCCL_MOVE(eptr));
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void set_stopped() noexcept final override
|
||||
{
|
||||
execution::set_stopped(_CCCL_MOVE(__rcvr_));
|
||||
}
|
||||
|
||||
protected:
|
||||
_CCCL_HOST_DEVICE_API auto query(const get_stop_token_t&) const noexcept -> inplace_stop_token final override
|
||||
{
|
||||
if constexpr (__callable<const get_stop_token_t&, env_of_t<_Rcvr>>)
|
||||
{
|
||||
if constexpr (__same_as<stop_token_of_t<env_of_t<_Rcvr>>, inplace_stop_token>)
|
||||
{
|
||||
return get_stop_token(get_env(__rcvr_));
|
||||
}
|
||||
}
|
||||
return inplace_stop_token{}; // MSVC thinks this is unreachable. :-?
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto query(const get_allocator_t&) const noexcept
|
||||
-> any_allocator<::cuda::std::byte> final override
|
||||
{
|
||||
return any_allocator{get_allocator(get_env(__rcvr_))};
|
||||
}
|
||||
|
||||
// defined in task_scheduler.cuh:
|
||||
_CCCL_HOST_DEVICE_API auto query(const get_scheduler_t& __query) const noexcept -> task_scheduler final override;
|
||||
|
||||
_Rcvr __rcvr_;
|
||||
};
|
||||
|
||||
template <class _Rcvr>
|
||||
struct __receiver_proxy : __receiver_proxy_base<_Rcvr, receiver_proxy>
|
||||
{
|
||||
using __receiver_proxy_base<_Rcvr, receiver_proxy>::__receiver_proxy_base;
|
||||
|
||||
_CCCL_HOST_DEVICE_API void set_value() noexcept final override
|
||||
{
|
||||
execution::set_value(_CCCL_MOVE(this->__rcvr_));
|
||||
}
|
||||
};
|
||||
|
||||
// A receiver type that forwards its completion operations to a _RcvrProxy member held by
|
||||
// reference (where _RcvrProxy is one of receiver_proxy or bulk_item_receiver_proxy). It
|
||||
// is also responsible to destroying and, if necessary, deallocating the operation state.
|
||||
template <class _RcvrProxy>
|
||||
struct __proxy_receiver
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
using __delete_fn_t = void(void*) noexcept;
|
||||
|
||||
_CCCL_HOST_DEVICE_API void set_value() noexcept
|
||||
{
|
||||
auto& __proxy = __rcvr_proxy_;
|
||||
__delete_fn_(__opstate_storage_);
|
||||
__proxy.set_value();
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void set_error(exception_ptr eptr) noexcept
|
||||
{
|
||||
auto& __proxy = __rcvr_proxy_;
|
||||
__delete_fn_(__opstate_storage_);
|
||||
__proxy.set_error(_CCCL_MOVE(eptr));
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void set_stopped() noexcept
|
||||
{
|
||||
auto& __proxy = __rcvr_proxy_;
|
||||
__delete_fn_(__opstate_storage_);
|
||||
__proxy.set_stopped();
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto get_env() const noexcept -> env_of_t<_RcvrProxy>
|
||||
{
|
||||
return execution::get_env(__rcvr_proxy_);
|
||||
}
|
||||
|
||||
_RcvrProxy& __rcvr_proxy_;
|
||||
void* __opstate_storage_;
|
||||
__delete_fn_t* __delete_fn_;
|
||||
};
|
||||
} // namespace __detail
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_PARALLEL_SCHEDULER
|
||||
@@ -1,137 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX___EXECUTION_POLICY_CUH
|
||||
#define __CUDAX___EXECUTION_POLICY_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__execution/env.h>
|
||||
#include <cuda/std/__execution/policy.h>
|
||||
#include <cuda/std/__type_traits/is_convertible.h>
|
||||
#include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
using ::cuda::std::execution::__execution_policy;
|
||||
using ::cuda::std::execution::par;
|
||||
using ::cuda::std::execution::par_unseq;
|
||||
using ::cuda::std::execution::seq;
|
||||
using ::cuda::std::execution::unseq;
|
||||
|
||||
struct any_execution_policy
|
||||
{
|
||||
using type = any_execution_policy;
|
||||
using value_type = __execution_policy;
|
||||
|
||||
_CCCL_HIDE_FROM_ABI any_execution_policy() = default;
|
||||
|
||||
template <uint32_t _Policy>
|
||||
_CCCL_HOST_API constexpr any_execution_policy(::cuda::std::execution::__execution_policy_base<_Policy>) noexcept
|
||||
: value(value_type{_Policy})
|
||||
{}
|
||||
|
||||
_CCCL_HOST_API constexpr operator __execution_policy() const noexcept
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
_CCCL_HOST_API constexpr auto operator()() const noexcept -> value_type
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
template <uint32_t _Policy>
|
||||
[[nodiscard]] _CCCL_HOST_API friend constexpr bool
|
||||
operator==(const any_execution_policy& pol, const ::cuda::std::execution::__execution_policy_base<_Policy>&) noexcept
|
||||
{
|
||||
return pol.value == value_type{_Policy};
|
||||
}
|
||||
|
||||
#if _CCCL_STD_VER <= 2017
|
||||
template <uint32_t _Policy>
|
||||
[[nodiscard]] _CCCL_HOST_API friend constexpr bool
|
||||
operator==(const ::cuda::std::execution::__execution_policy_base<_Policy>&, const any_execution_policy& pol) noexcept
|
||||
{
|
||||
return pol.value == value_type{_Policy};
|
||||
}
|
||||
|
||||
template <uint32_t _Policy>
|
||||
[[nodiscard]] _CCCL_HOST_API friend constexpr bool
|
||||
operator!=(const any_execution_policy& pol, const ::cuda::std::execution::__execution_policy_base<_Policy>&) noexcept
|
||||
{
|
||||
return pol.value != value_type{_Policy};
|
||||
}
|
||||
|
||||
template <uint32_t _Policy>
|
||||
[[nodiscard]] _CCCL_HOST_API friend constexpr bool
|
||||
operator!=(const ::cuda::std::execution::__execution_policy_base<_Policy>&, const any_execution_policy& pol)
|
||||
{
|
||||
return pol.value != value_type{_Policy};
|
||||
}
|
||||
#endif // _CCCL_STD_VER <= 2017
|
||||
|
||||
__execution_policy value = __execution_policy::__invalid_execution_policy;
|
||||
};
|
||||
|
||||
struct get_execution_policy_t;
|
||||
|
||||
template <class _Tp>
|
||||
_CCCL_CONCEPT __has_member_get_execution_policy = _CCCL_REQUIRES_EXPR((_Tp), const _Tp& __t)(
|
||||
requires(::cuda::std::is_convertible_v<decltype(__t.get_execution_policy()), __execution_policy>));
|
||||
|
||||
template <class _Env>
|
||||
_CCCL_CONCEPT __has_query_get_execution_policy = _CCCL_REQUIRES_EXPR((_Env))(
|
||||
requires(!__has_member_get_execution_policy<_Env>),
|
||||
requires(::cuda::std::is_convertible_v<::cuda::std::execution::__query_result_t<const _Env&, get_execution_policy_t>,
|
||||
__execution_policy>));
|
||||
|
||||
struct get_execution_policy_t
|
||||
{
|
||||
_CCCL_TEMPLATE(class _Tp)
|
||||
_CCCL_REQUIRES(__has_member_get_execution_policy<_Tp>)
|
||||
[[nodiscard]] _CCCL_HIDE_FROM_ABI auto operator()(const _Tp& __t) const noexcept
|
||||
{
|
||||
return __t.get_execution_policy();
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Env)
|
||||
_CCCL_REQUIRES(__has_query_get_execution_policy<_Env>)
|
||||
[[nodiscard]] _CCCL_HIDE_FROM_ABI auto operator()(const _Env& __env) const noexcept
|
||||
{
|
||||
static_assert(noexcept(__env.query(*this)));
|
||||
return __env.query(*this);
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT get_execution_policy_t get_execution_policy{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
template <>
|
||||
inline constexpr bool is_execution_policy_v<::cuda::experimental::execution::any_execution_policy> = true;
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif //__CUDAX___EXECUTION_POLICY_CUH
|
||||
@@ -1,43 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
// IMPORTANT: This file intionally lacks a header guard.
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CUDAX_ASYNC_PROLOGUE_INCLUDED)
|
||||
# error multiple inclusion of prologue.cuh
|
||||
#endif
|
||||
|
||||
#define _CUDAX_ASYNC_PROLOGUE_INCLUDED
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wsubobject-linkage")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-value")
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(4714) // function 'foo' marked as __forceinline not inlined
|
||||
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wmissing-braces")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wmissing-braces")
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(5246) // missing braces around initializer
|
||||
|
||||
#if _CCCL_CUDA_COMPILER(NVHPC)
|
||||
_CCCL_BEGIN_NV_DIAG_SUPPRESS(cuda_compile)
|
||||
#endif // _CCCL_CUDA_COMPILER(NVHPC)
|
||||
|
||||
// private and protected nested class types cannot be used as tparams to __global__
|
||||
// functions. _CUDAX_SEMI_PRIVATE expands to public when _CCCL_CUDA_COMPILATION() is true,
|
||||
// and private otherwise.
|
||||
#if _CCCL_CUDA_COMPILATION()
|
||||
# define _CUDAX_SEMI_PRIVATE public
|
||||
#else // ^^^ _CCCL_CUDA_COMPILATION() ^^^ / vvv !_CCCL_CUDA_COMPILATION() vvv
|
||||
# define _CUDAX_SEMI_PRIVATE private
|
||||
#endif
|
||||
@@ -1,409 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_QUERIES
|
||||
#define __CUDAX_EXECUTION_QUERIES
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
_CCCL_SUPPRESS_DEPRECATED_PUSH
|
||||
_CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
|
||||
#include <cuda/std/__memory/allocator.h>
|
||||
_CCCL_SUPPRESS_DEPRECATED_POP
|
||||
|
||||
#include <cuda/__launch/configuration.h>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__concepts/convertible_to.h>
|
||||
#include <cuda/std/__execution/env.h>
|
||||
#include <cuda/std/__type_traits/enable_if.h>
|
||||
#include <cuda/std/__type_traits/is_callable.h>
|
||||
#include <cuda/std/__type_traits/remove_cvref.h>
|
||||
#include <cuda/std/__utility/exchange.h>
|
||||
#include <cuda/std/__utility/unreachable.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/completion_behavior.cuh>
|
||||
#include <cuda/experimental/__execution/domain.cuh>
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/meta.cuh>
|
||||
#include <cuda/experimental/__execution/stop_token.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
namespace __detail
|
||||
{
|
||||
template <class _Env, class _Query>
|
||||
using __statically_queryable_with_t = decltype(::cuda::std::remove_cvref_t<_Env>::query(declval<_Query>()));
|
||||
} // namespace __detail
|
||||
|
||||
template <class _Env, class _Query>
|
||||
_CCCL_CONCEPT __statically_queryable_with =
|
||||
__is_instantiable_with<__detail::__statically_queryable_with_t, _Env, _Query>;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// get_allocator
|
||||
_CCCL_GLOBAL_CONSTANT struct get_allocator_t
|
||||
{
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Env& __env) const noexcept
|
||||
-> __query_result_or_t<_Env, get_allocator_t, ::cuda::std::allocator<::cuda::std::byte>>
|
||||
{
|
||||
static_assert(__nothrow_queryable_with_or<_Env, get_allocator_t, true>,
|
||||
"The get_allocator query must be noexcept.");
|
||||
// NOT TO SPEC: return a default allocator if the query is not supported.
|
||||
return __query_or(__env, *this, ::cuda::std::allocator<::cuda::std::byte>{});
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto query(forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
} get_allocator{};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// get_stop_token
|
||||
_CCCL_GLOBAL_CONSTANT struct get_stop_token_t
|
||||
{
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Env& __env) const noexcept
|
||||
-> __query_result_or_t<_Env, get_stop_token_t, never_stop_token>
|
||||
{
|
||||
static_assert(__nothrow_queryable_with_or<_Env, get_stop_token_t, true>,
|
||||
"The get_stop_token query must be noexcept.");
|
||||
return __query_or(__env, *this, never_stop_token{});
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto query(forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
} get_stop_token{};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// get_scheduler
|
||||
_CCCL_GLOBAL_CONSTANT struct get_scheduler_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Tag = set_value_t, class _Env)
|
||||
_CCCL_REQUIRES(__queryable_with<_Env, get_scheduler_t>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Env& __env) const noexcept
|
||||
-> __call_result_t<get_completion_scheduler_t<_Tag>,
|
||||
__query_result_t<_Env, get_scheduler_t>,
|
||||
__hide_scheduler<const _Env&>>
|
||||
{
|
||||
static_assert(noexcept(__env.query(*this)));
|
||||
static_assert(__is_scheduler<__query_result_t<_Env, get_scheduler_t>>);
|
||||
return get_completion_scheduler_t<_Tag>()(__env.query(*this), __hide_scheduler{__env});
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto query(forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
} get_scheduler{};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// get_delegation_scheduler
|
||||
_CCCL_GLOBAL_CONSTANT struct get_delegation_scheduler_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Env)
|
||||
_CCCL_REQUIRES(__queryable_with<_Env, get_delegation_scheduler_t>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Env& __env) const noexcept
|
||||
-> __query_result_t<_Env, get_delegation_scheduler_t>
|
||||
{
|
||||
static_assert(noexcept(__env.query(*this)));
|
||||
static_assert(__is_scheduler<decltype(__env.query(*this))>);
|
||||
return __env.query(*this);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto query(forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
} get_delegation_scheduler{};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// get_completion_scheduler
|
||||
|
||||
//! @brief A query type for asking a sender's attributes for the scheduler on which that
|
||||
//! sender will complete.
|
||||
//!
|
||||
//! @tparam _Tag one of set_value_t, set_error_t, or set_stopped_t
|
||||
template <class _Tag>
|
||||
struct get_completion_scheduler_t
|
||||
{
|
||||
// This function object reads the completion scheduler from an attribute object or a
|
||||
// scheduler, accounting for the fact that the query member function may or may not
|
||||
// accept an environment.
|
||||
struct __read_query_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Attrs, class _GetComplSch = get_completion_scheduler_t)
|
||||
_CCCL_REQUIRES(__queryable_with<_Attrs, _GetComplSch>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
operator()(const _Attrs& __attrs, cuda::std::__ignore_t = {}) const noexcept
|
||||
-> decay_t<__query_result_t<_Attrs, _GetComplSch>>
|
||||
{
|
||||
static_assert(noexcept(__attrs.query(_GetComplSch{})));
|
||||
static_assert(__is_scheduler<decltype(__attrs.query(_GetComplSch{}))>,
|
||||
"The get_completion_scheduler query must return a scheduler type.");
|
||||
return __attrs.query(_GetComplSch{});
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Attrs, class _Env, class _GetComplSch = get_completion_scheduler_t)
|
||||
_CCCL_REQUIRES(__queryable_with<_Attrs, _GetComplSch, const _Env&>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
operator()(const _Attrs& __attrs, const _Env& __env) const noexcept
|
||||
-> decay_t<__query_result_t<_Attrs, _GetComplSch, const _Env&>>
|
||||
{
|
||||
static_assert(noexcept(__attrs.query(_GetComplSch{}, __env)));
|
||||
static_assert(__is_scheduler<decltype(__attrs.query(_GetComplSch{}, __env))>,
|
||||
"The get_completion_scheduler query must return a scheduler type.");
|
||||
return __attrs.query(_GetComplSch{}, __env);
|
||||
}
|
||||
};
|
||||
|
||||
private:
|
||||
// A scheduler might have a completion scheduler different from itself; for example, an
|
||||
// inline_scheduler completes wherever the scheduler's sender is started. So we
|
||||
// recursively ask the scheduler for its completion scheduler until we find one whose
|
||||
// completion scheduler is equal to itself (or it doesn't have one).
|
||||
struct __recurse_query_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Self = __recurse_query_t, class _Sch, class... _Env>
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()([[maybe_unused]] _Sch __sch, const _Env&... __env) const noexcept
|
||||
{
|
||||
// When determining where the scheduler's operations will complete, we query
|
||||
// for the completion scheduler of the value channel:
|
||||
using __read_query_t = typename get_completion_scheduler_t<set_value_t>::__read_query_t;
|
||||
|
||||
if constexpr (__callable<__read_query_t, _Sch, const _Env&...>)
|
||||
{
|
||||
using __sch2_t = decay_t<__call_result_t<__read_query_t, _Sch, const _Env&...>>;
|
||||
if constexpr (__same_as<_Sch, __sch2_t>)
|
||||
{
|
||||
_Sch __prev = __sch;
|
||||
do
|
||||
{
|
||||
__prev = cuda::std::exchange(__sch, __read_query_t{}(__sch, __env...));
|
||||
} while (__prev != __sch);
|
||||
return __sch;
|
||||
}
|
||||
else
|
||||
{
|
||||
// New scheduler has different type. Recurse!
|
||||
return _Self{}(__read_query_t{}(__sch, __env...), __env...);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (__callable<__read_query_t, env_of_t<schedule_result_t<_Sch>>, const _Env&...>)
|
||||
{
|
||||
_CCCL_ASSERT(__sch == __read_query_t{}(get_env(__sch.schedule()), __env...),
|
||||
"the scheduler's sender must have a completion scheduler attribute equal to the scheduler that "
|
||||
"provided it.");
|
||||
}
|
||||
return __sch;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Attrs, class... _Env, class _Sch>
|
||||
[[nodiscard]] _CCCL_TRIVIAL_API constexpr static auto __check_domain(_Sch __sch) noexcept -> _Sch
|
||||
{
|
||||
// Sanity check: if a completion domain can be determined, then it must match the
|
||||
// domain of the completion scheduler.
|
||||
if constexpr (__callable<get_completion_domain_t<_Tag>, const _Attrs&, const _Env&...>)
|
||||
{
|
||||
using __domain_t = __call_result_t<get_completion_domain_t<_Tag>, const _Attrs&, const _Env&...>;
|
||||
static_assert(__same_as<__domain_t, __scheduler_domain_t<_Sch, const _Env&...>>,
|
||||
"the sender claims to complete on a domain that is not the domain of its completion scheduler");
|
||||
}
|
||||
return __sch;
|
||||
}
|
||||
|
||||
template <class _Attrs, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto __get_declfn() noexcept
|
||||
{
|
||||
// If __attrs has a completion scheduler, then return it (after checking the scheduler
|
||||
// for _its_ completion scheduler):
|
||||
if constexpr (__callable<__read_query_t, const _Attrs&, const _Env&...>)
|
||||
{
|
||||
using __result_t =
|
||||
decltype(__recurse_query_t{}(__read_query_t{}(declval<_Attrs>(), declval<_Env>()...), declval<_Env>()...));
|
||||
return __declfn<__result_t>;
|
||||
}
|
||||
// Otherwise, if __attrs indicates that its sender completes inline, then we can ask
|
||||
// the environment for the current scheduler and return that (after checking the
|
||||
// scheduler for _its_ completion scheduler).
|
||||
else if constexpr (__completes_inline<_Attrs, _Env...> && __callable<get_scheduler_t, const _Env&...>)
|
||||
{
|
||||
using __result_t =
|
||||
decltype(__recurse_query_t{}(get_scheduler(declval<_Env>()...), __hide_scheduler{declval<_Env>()}...));
|
||||
return __declfn<__result_t>;
|
||||
}
|
||||
else if constexpr (__is_scheduler<_Attrs> && sizeof...(_Env) != 0)
|
||||
{
|
||||
return __declfn<decay_t<_Attrs>>;
|
||||
}
|
||||
// Otherwise, no completion scheduler can be determined. Return void.
|
||||
}
|
||||
|
||||
public:
|
||||
template <class _Attrs, class... _Env, auto _DeclFn = __get_declfn<const _Attrs&, const _Env&...>()>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
operator()(const _Attrs& __attrs, const _Env&... __env) const noexcept -> __unless_one_of_t<decltype(_DeclFn()), void>
|
||||
{
|
||||
// If __attrs has a completion scheduler, then return it (after checking the scheduler
|
||||
// for _its_ completion scheduler):
|
||||
if constexpr (__callable<__read_query_t, const _Attrs&, const _Env&...>)
|
||||
{
|
||||
return __check_domain<_Attrs, _Env...>(__recurse_query_t{}(__read_query_t{}(__attrs, __env...), __env...));
|
||||
}
|
||||
// Otherwise, if __attrs indicates that its sender completes inline, then we can ask
|
||||
// the environment for the current scheduler and return that (after checking the
|
||||
// scheduler for _its_ completion scheduler).
|
||||
else if constexpr (__completes_inline<_Attrs, _Env...> && __callable<get_scheduler_t, const _Env&...>)
|
||||
{
|
||||
return __check_domain<_Attrs, _Env...>(__recurse_query_t{}(get_scheduler(__env...), __hide_scheduler{__env}...));
|
||||
}
|
||||
else
|
||||
{
|
||||
return __attrs;
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto query(forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Tag>
|
||||
extern ::cuda::std::__undefined<_Tag> get_completion_scheduler;
|
||||
|
||||
// Explicitly instantiate these because of variable template weirdness in device code
|
||||
template <>
|
||||
_CCCL_GLOBAL_CONSTANT get_completion_scheduler_t<set_value_t> get_completion_scheduler<set_value_t>{};
|
||||
template <>
|
||||
_CCCL_GLOBAL_CONSTANT get_completion_scheduler_t<set_error_t> get_completion_scheduler<set_error_t>{};
|
||||
template <>
|
||||
_CCCL_GLOBAL_CONSTANT get_completion_scheduler_t<set_stopped_t> get_completion_scheduler<set_stopped_t>{};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// __is_completion_query
|
||||
template <class _Query>
|
||||
inline constexpr bool __is_completion_query = false;
|
||||
template <class _Tag>
|
||||
inline constexpr bool __is_completion_query<get_completion_domain_t<_Tag>> = true;
|
||||
template <class _Tag>
|
||||
inline constexpr bool __is_completion_query<get_completion_scheduler_t<_Tag>> = true;
|
||||
template <>
|
||||
inline constexpr bool __is_completion_query<get_completion_behavior_t> = true;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// get_forward_progress_guarantee
|
||||
|
||||
// This query is not a forwarding query.
|
||||
_CCCL_GLOBAL_CONSTANT struct get_forward_progress_guarantee_t
|
||||
{
|
||||
template <class _Sch>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Sch& __sch) const noexcept
|
||||
-> forward_progress_guarantee
|
||||
{
|
||||
static_assert(__nothrow_queryable_with_or<_Sch, get_forward_progress_guarantee_t, true>,
|
||||
"The get_forward_progress_guarantee query must be noexcept.");
|
||||
return __query_or(__sch, *this, forward_progress_guarantee::weakly_parallel);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto query(forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return false;
|
||||
}
|
||||
} get_forward_progress_guarantee{};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// get_available_parallelism
|
||||
|
||||
// This query is not a forwarding query.
|
||||
_CCCL_GLOBAL_CONSTANT struct get_available_parallelism_t
|
||||
{
|
||||
template <class _Sch>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Sch& __sch) const noexcept
|
||||
{
|
||||
static_assert(__nothrow_queryable_with_or<const _Sch&, get_available_parallelism_t, true>,
|
||||
"The get_available_parallelism query must be noexcept.");
|
||||
static_assert(
|
||||
cuda::std::convertible_to<__query_result_or_t<const _Sch&, get_available_parallelism_t, size_t>, size_t>,
|
||||
"The get_available_parallelism query must return a type convertible to size_t.");
|
||||
return __query_or(__sch, *this, size_t(1));
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_NODEBUG_API static constexpr auto query(forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return false;
|
||||
}
|
||||
} get_available_parallelism{};
|
||||
|
||||
// By default, CUDA kernels are launched with a single thread and a single block.
|
||||
using __single_threaded_config_base_t = decltype(make_config(grid_dims<1>(), block_dims<1>()));
|
||||
|
||||
// We hide the complicated type of the default launch configuration so diagnositics are
|
||||
// easier to read.
|
||||
struct __single_threaded_config_t : __single_threaded_config_base_t
|
||||
{
|
||||
_CCCL_HOST_API constexpr __single_threaded_config_t() noexcept
|
||||
: __single_threaded_config_base_t{make_config(grid_dims<1>(), block_dims<1>())}
|
||||
{}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT __single_threaded_config_t __single_threaded_config{};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// get_launch_config: A sender can define this attribute to control the launch configuration
|
||||
// of the kernel it will launch when executed on a CUDA stream scheduler.
|
||||
_CCCL_GLOBAL_CONSTANT struct get_launch_config_t
|
||||
{
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Env& __env) const noexcept
|
||||
-> __query_result_or_t<_Env, get_launch_config_t, __single_threaded_config_t>
|
||||
{
|
||||
static_assert(__nothrow_queryable_with_or<_Env, get_launch_config_t, true>,
|
||||
"The get_launch_config query must be noexcept.");
|
||||
return __query_or(__env, *this, __single_threaded_config);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto query(forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
} get_launch_config{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_QUERIES
|
||||
@@ -1,105 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_RCVR_REF
|
||||
#define __CUDAX_EXECUTION_RCVR_REF
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__type_traits/is_specialization_of.h>
|
||||
#include <cuda/std/__memory/addressof.h>
|
||||
#include <cuda/std/__type_traits/is_copy_constructible.h>
|
||||
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
_CCCL_BEGIN_NV_DIAG_SUPPRESS(114) // function "foo" was referenced but not defined
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
template <class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_ref
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
_CCCL_HOST_DEVICE_API explicit constexpr __rcvr_ref(_Rcvr& __rcvr) noexcept
|
||||
: __rcvr_{::cuda::std::addressof(__rcvr)}
|
||||
{}
|
||||
|
||||
template <class... _As>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_value(_As&&... __as) noexcept
|
||||
{
|
||||
execution::set_value(static_cast<_Rcvr&&>(*__rcvr_), static_cast<_As&&>(__as)...);
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_error(_Error&& __err) noexcept
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(*__rcvr_), static_cast<_Error&&>(__err));
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_stopped() noexcept
|
||||
{
|
||||
execution::set_stopped(static_cast<_Rcvr&&>(*__rcvr_));
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> env_of_t<_Rcvr>
|
||||
{
|
||||
return execution::get_env(*__rcvr_);
|
||||
}
|
||||
|
||||
private:
|
||||
_Rcvr* __rcvr_;
|
||||
};
|
||||
|
||||
// The __ref_rcvr function and its helpers are used to avoid wrapping a receiver in a
|
||||
// __rcvr_ref when that is possible. The logic goes as follows:
|
||||
//
|
||||
// 1. If the receiver is an instance of __rcvr_ref, return it.
|
||||
// 2. If the receiver is nothrow copy constructible, return it.
|
||||
// 3. Otherwise, return a __rcvr_ref wrapping the receiver.
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __ref_rcvr(_Rcvr& __rcvr) noexcept
|
||||
{
|
||||
if constexpr (__is_specialization_of_v<_Rcvr, __rcvr_ref>)
|
||||
{
|
||||
return __rcvr;
|
||||
}
|
||||
else if constexpr (__nothrow_constructible<_Rcvr, const _Rcvr&>)
|
||||
{
|
||||
return const_cast<const _Rcvr&>(__rcvr);
|
||||
}
|
||||
else
|
||||
{
|
||||
return __rcvr_ref{__rcvr};
|
||||
}
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
using __rcvr_ref_t _CCCL_NODEBUG_ALIAS = decltype(execution::__ref_rcvr(::cuda::std::declval<_Rcvr&>()));
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
_CCCL_END_NV_DIAG_SUPPRESS() // function "foo" was references but not defined
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_RCVR_REF
|
||||
@@ -1,103 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the _Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: _Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_RCVR_WITH_ENV
|
||||
#define __CUDAX_EXECUTION_RCVR_WITH_ENV
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
template <class _Rcvr, class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_with_env_t;
|
||||
|
||||
// If _Env has a value for the `get_scheduler` query, then we must ensure that we report
|
||||
// the domain correctly. Under no circumstances should we forward the `get_domain` query
|
||||
// to the receiver's environment. That environment may have a domain that does not
|
||||
// conform to the scheduler in _Env.
|
||||
template <class _Env, class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __env_with_rcvr_t
|
||||
{
|
||||
// Prefer to query _Env
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Query, class... _Args)
|
||||
_CCCL_REQUIRES(__queryable_with<_Env, _Query, _Args...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(_Query, _Args&&... __args) const
|
||||
noexcept(__nothrow_queryable_with<_Env, _Query, _Args...>) -> __query_result_t<_Env, _Query, _Args...>
|
||||
{
|
||||
return __rcvr_->__env_.query(_Query{}, static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
// Fallback to querying the inner receiver's environment, but only for forwarding
|
||||
// queries.
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Query, class... _Args)
|
||||
_CCCL_REQUIRES((!__queryable_with<_Env, _Query, _Args...>)
|
||||
_CCCL_AND __forwarding_query<_Query> _CCCL_AND __queryable_with<env_of_t<_Rcvr>, _Query, _Args...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(_Query, _Args&&... __args) const
|
||||
noexcept(__nothrow_queryable_with<env_of_t<_Rcvr>, _Query, _Args...>)
|
||||
-> __query_result_t<env_of_t<_Rcvr>, _Query, _Args...>
|
||||
{
|
||||
// If _Env has a value for the `get_scheduler` query, then we should not be
|
||||
// forwarding a get_domain query to the parent receiver's environment.
|
||||
static_assert(!__same_as<_Query, get_domain_t> || !__queryable_with<_Env, get_scheduler_t>,
|
||||
"_Env specifies a scheduler but not a domain.");
|
||||
return execution::get_env(__rcvr_->__base()).query(_Query{}, static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
__rcvr_with_env_t<_Rcvr, _Env> const* __rcvr_;
|
||||
};
|
||||
|
||||
template <class _Rcvr, class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_with_env_t : _Rcvr
|
||||
{
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto __base() && noexcept -> _Rcvr&&
|
||||
{
|
||||
return static_cast<_Rcvr&&>(*this);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto __base() & noexcept -> _Rcvr&
|
||||
{
|
||||
return *this;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto __base() const& noexcept -> _Rcvr const&
|
||||
{
|
||||
return *this;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __env_with_rcvr_t<_Env, _Rcvr>
|
||||
{
|
||||
return __env_with_rcvr_t<_Env, _Rcvr>{this};
|
||||
}
|
||||
|
||||
_Env __env_;
|
||||
};
|
||||
|
||||
template <class _Rcvr, class _Env>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES __rcvr_with_env_t(_Rcvr, _Env) -> __rcvr_with_env_t<_Rcvr, _Env>;
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_RCVR_WITH_ENV
|
||||
@@ -1,169 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_READ_ENV
|
||||
#define __CUDAX_EXECUTION_READ_ENV
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__cccl/unreachable.h>
|
||||
#include <cuda/std/__exception/exception_macros.h>
|
||||
#include <cuda/std/__type_traits/is_callable.h>
|
||||
#include <cuda/std/__type_traits/is_void.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/exception.cuh>
|
||||
#include <cuda/experimental/__execution/get_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/queries.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
struct _THE_CURRENT_ENVIRONMENT_LACKS_THIS_QUERY;
|
||||
struct _THE_CURRENT_ENVIRONMENT_RETURNED_VOID_FOR_THIS_QUERY;
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT read_env_t
|
||||
{
|
||||
private:
|
||||
template <class _Rcvr, class _Query>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
|
||||
_Rcvr __rcvr_;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __opstate_t(_Rcvr __rcvr) noexcept
|
||||
: __rcvr_(static_cast<_Rcvr&&>(__rcvr))
|
||||
{}
|
||||
|
||||
_CCCL_IMMOVABLE(__opstate_t);
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_HOST_DEVICE_API void start() noexcept
|
||||
{
|
||||
// If the query invocation is noexcept, call it directly. Otherwise,
|
||||
// wrap it in a try-catch block and forward the exception to the
|
||||
// receiver.
|
||||
if constexpr (__nothrow_callable<_Query, env_of_t<_Rcvr>>)
|
||||
{
|
||||
// This looks like a use after move, but `set_value` takes its
|
||||
// arguments by forwarding reference, so it's safe.
|
||||
execution::set_value(static_cast<_Rcvr&&>(__rcvr_), _Query{}(execution::get_env(__rcvr_)));
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
execution::set_value(static_cast<_Rcvr&&>(__rcvr_), _Query{}(execution::get_env(__rcvr_)));
|
||||
}
|
||||
_CCCL_CATCH_ALL
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(__rcvr_), execution::current_exception());
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
struct __attrs_t
|
||||
{
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_behavior_t) const noexcept
|
||||
{
|
||||
return completion_behavior::inline_completion;
|
||||
}
|
||||
};
|
||||
|
||||
public:
|
||||
template <class _Query>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
/// @brief Returns a sender that, when connected to a receiver and started,
|
||||
/// invokes the query with the receiver's environment and forwards the result
|
||||
/// to the receiver's `set_value` member.
|
||||
template <class _Query>
|
||||
_CCCL_HOST_DEVICE_API constexpr __sndr_t<_Query> operator()(_Query) const noexcept;
|
||||
};
|
||||
|
||||
template <class _Query>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT read_env_t::__sndr_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class _Self, class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
if constexpr (!__callable<_Query, _Env>)
|
||||
{
|
||||
return invalid_completion_signature<_WHERE(_IN_ALGORITHM, read_env_t),
|
||||
_WHAT(_THE_CURRENT_ENVIRONMENT_LACKS_THIS_QUERY),
|
||||
_WITH_QUERY(_Query),
|
||||
_WITH_ENVIRONMENT(_Env)>();
|
||||
}
|
||||
else if constexpr (::cuda::std::is_void_v<__call_result_t<_Query, _Env>>)
|
||||
{
|
||||
return invalid_completion_signature<_WHERE(_IN_ALGORITHM, read_env_t),
|
||||
_WHAT(_THE_CURRENT_ENVIRONMENT_RETURNED_VOID_FOR_THIS_QUERY),
|
||||
_WITH_QUERY(_Query),
|
||||
_WITH_ENVIRONMENT(_Env)>();
|
||||
}
|
||||
else
|
||||
{
|
||||
return completion_signatures<set_value_t(__call_result_t<_Query, _Env>)>{}
|
||||
+ __eptr_completion_if<!__nothrow_callable<_Query, _Env>>();
|
||||
}
|
||||
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) const noexcept -> __opstate_t<_Rcvr, _Query>
|
||||
{
|
||||
return __opstate_t<_Rcvr, _Query>{static_cast<_Rcvr&&>(__rcvr)};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto get_env() noexcept
|
||||
{
|
||||
return __attrs_t{};
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ read_env_t __tag;
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ _Query __query;
|
||||
};
|
||||
|
||||
template <class _Query>
|
||||
_CCCL_HOST_DEVICE_API constexpr read_env_t::__sndr_t<_Query> read_env_t::operator()(_Query __query) const noexcept
|
||||
{
|
||||
return __sndr_t<_Query>{{}, __query};
|
||||
}
|
||||
|
||||
template <class _Query>
|
||||
inline constexpr int structured_binding_size<read_env_t::__sndr_t<_Query>> = 2;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT read_env_t read_env{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_READ_ENV
|
||||
@@ -1,313 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_RUN_LOOP
|
||||
#define __CUDAX_EXECUTION_RUN_LOOP
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__utility/immovable.h>
|
||||
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/atomic_intrusive_queue.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/exception.cuh>
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/queries.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT __run_loop_base : __immovable
|
||||
{
|
||||
public:
|
||||
_CCCL_HIDE_FROM_ABI __run_loop_base() = default;
|
||||
|
||||
_CCCL_HOST_DEVICE_API void run() noexcept
|
||||
{
|
||||
// execute work items until the __finishing_ flag is set:
|
||||
while (!__finishing_.load(::cuda::std::memory_order_acquire))
|
||||
{
|
||||
__queue_.wait_for_item();
|
||||
__execute_all();
|
||||
}
|
||||
// drain the queue, taking care to execute any tasks that get added while
|
||||
// executing the remaining tasks:
|
||||
while (__execute_all())
|
||||
;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void finish() noexcept
|
||||
{
|
||||
if (!__finishing_.exchange(true, ::cuda::std::memory_order_acq_rel))
|
||||
{
|
||||
// push an empty work item to the queue to wake up the consuming thread
|
||||
// and let it finish:
|
||||
__queue_.push(&__noop_task);
|
||||
}
|
||||
}
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __task : __immovable
|
||||
{
|
||||
using __execute_fn_t _CCCL_NODEBUG_ALIAS = void(__task*) noexcept;
|
||||
|
||||
_CCCL_HIDE_FROM_ABI __task() = default;
|
||||
_CCCL_HOST_DEVICE_API explicit __task(__execute_fn_t* __execute_fn) noexcept
|
||||
: __execute_fn_(__execute_fn)
|
||||
{}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void __execute() noexcept
|
||||
{
|
||||
(*__execute_fn_)(this);
|
||||
}
|
||||
|
||||
__execute_fn_t* __execute_fn_ = nullptr;
|
||||
__task* __next_ = nullptr;
|
||||
};
|
||||
|
||||
template <class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t : __task
|
||||
{
|
||||
__atomic_intrusive_queue<&__task::__next_>* __queue_;
|
||||
_Rcvr __rcvr_;
|
||||
|
||||
_CCCL_HOST_DEVICE_API static void __execute_impl(__task* __p) noexcept
|
||||
{
|
||||
static_assert(noexcept(get_stop_token(declval<env_of_t<_Rcvr>>()).stop_requested()));
|
||||
auto& __rcvr = static_cast<__opstate_t*>(__p)->__rcvr_;
|
||||
|
||||
if (get_stop_token(get_env(__rcvr)).stop_requested())
|
||||
{
|
||||
set_stopped(static_cast<_Rcvr&&>(__rcvr));
|
||||
}
|
||||
else
|
||||
{
|
||||
set_value(static_cast<_Rcvr&&>(__rcvr));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __opstate_t(
|
||||
__atomic_intrusive_queue<&__task::__next_>* __queue, _Rcvr __rcvr)
|
||||
: __task{&__execute_impl}
|
||||
, __queue_{__queue}
|
||||
, __rcvr_{static_cast<_Rcvr&&>(__rcvr)}
|
||||
{}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void start() noexcept
|
||||
{
|
||||
__queue_->push(this);
|
||||
}
|
||||
};
|
||||
|
||||
// Returns true if any tasks were executed.
|
||||
_CCCL_HOST_DEVICE_API bool __execute_all() noexcept
|
||||
{
|
||||
// Dequeue all tasks at once. This returns an __intrusive_queue.
|
||||
auto __queue = __queue_.pop_all();
|
||||
|
||||
// Execute all the tasks in the queue.
|
||||
auto __it = __queue.begin();
|
||||
if (__it == __queue.end())
|
||||
{
|
||||
return false; // No tasks to execute.
|
||||
}
|
||||
|
||||
do
|
||||
{
|
||||
// Take care to increment the iterator before executing the task,
|
||||
// because __execute() may invalidate the current node.
|
||||
auto __prev = __it++;
|
||||
(*__prev)->__execute();
|
||||
} while (__it != __queue.end());
|
||||
|
||||
__queue.clear();
|
||||
return true;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API static void __noop_(__task*) noexcept {}
|
||||
|
||||
::cuda::std::atomic<bool> __finishing_{false};
|
||||
__atomic_intrusive_queue<&__task::__next_> __queue_{};
|
||||
__task __noop_task{&__noop_};
|
||||
};
|
||||
|
||||
template <class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT basic_run_loop : __run_loop_base
|
||||
{
|
||||
private:
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __attrs_t
|
||||
{
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_scheduler_t<set_value_t>) const noexcept;
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_scheduler_t<set_stopped_t>) const noexcept;
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_domain_t<set_value_t>) const noexcept;
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_domain_t<set_stopped_t>) const noexcept;
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_behavior_t) const noexcept
|
||||
{
|
||||
return completion_behavior::asynchronous;
|
||||
}
|
||||
|
||||
basic_run_loop* __loop_;
|
||||
};
|
||||
|
||||
public:
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit basic_run_loop(_Env __env) noexcept
|
||||
: __env_{static_cast<_Env&&>(__env)}
|
||||
{}
|
||||
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT scheduler : __attrs_t
|
||||
{
|
||||
private:
|
||||
friend basic_run_loop;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit scheduler(basic_run_loop* __loop) noexcept
|
||||
: __attrs_t{__loop}
|
||||
{}
|
||||
|
||||
public:
|
||||
using scheduler_concept = scheduler_t;
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) const noexcept -> __opstate_t<_Rcvr>
|
||||
{
|
||||
return __opstate_t<_Rcvr>{&__loop_->__queue_, static_cast<_Rcvr&&>(__rcvr)};
|
||||
}
|
||||
|
||||
template <class _Self>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures() noexcept
|
||||
{
|
||||
return completion_signatures<set_value_t(), set_stopped_t()>{};
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __attrs_t
|
||||
{
|
||||
return __attrs_t{__loop_};
|
||||
}
|
||||
|
||||
private:
|
||||
friend scheduler;
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __sndr_t(basic_run_loop* __loop) noexcept
|
||||
: __loop_(__loop)
|
||||
{}
|
||||
|
||||
basic_run_loop* __loop_;
|
||||
};
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto schedule() const noexcept -> __sndr_t
|
||||
{
|
||||
return __sndr_t{this->__loop_};
|
||||
}
|
||||
|
||||
using __attrs_t::query;
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_forward_progress_guarantee_t) const noexcept
|
||||
-> forward_progress_guarantee
|
||||
{
|
||||
return forward_progress_guarantee::parallel;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
|
||||
operator==(const scheduler& __a, const scheduler& __b) noexcept
|
||||
{
|
||||
return __a.__loop_ == __b.__loop_;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
|
||||
operator!=(const scheduler& __a, const scheduler& __b) noexcept
|
||||
{
|
||||
return __a.__loop_ != __b.__loop_;
|
||||
}
|
||||
};
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_scheduler() noexcept -> scheduler
|
||||
{
|
||||
return scheduler{this};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> const _Env&
|
||||
{
|
||||
return __env_;
|
||||
}
|
||||
|
||||
private:
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ _Env __env_;
|
||||
};
|
||||
|
||||
// A run_loop with an empty environment. This is a struct instead of a type alias to give
|
||||
// it a simpler type name that is easier to read in diagnostics.
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT run_loop : basic_run_loop<env<>>
|
||||
{
|
||||
_CCCL_HIDE_FROM_ABI constexpr run_loop() noexcept
|
||||
: basic_run_loop<env<>>{env{}}
|
||||
{}
|
||||
};
|
||||
|
||||
template <class _Env>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto
|
||||
basic_run_loop<_Env>::__attrs_t::query(get_completion_scheduler_t<set_value_t>) const noexcept
|
||||
{
|
||||
if constexpr (__callable<get_scheduler_t, _Env&>)
|
||||
{
|
||||
return execution::get_scheduler(__loop_->__env_);
|
||||
}
|
||||
else
|
||||
{
|
||||
return scheduler{__loop_};
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Env>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto
|
||||
basic_run_loop<_Env>::__attrs_t::query(get_completion_scheduler_t<set_stopped_t>) const noexcept
|
||||
{
|
||||
return query(get_completion_scheduler<set_value_t>);
|
||||
}
|
||||
|
||||
template <class _Env>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto
|
||||
basic_run_loop<_Env>::__attrs_t::query(get_completion_domain_t<set_value_t>) const noexcept
|
||||
{
|
||||
if constexpr (__callable<get_domain_t, _Env&>)
|
||||
{
|
||||
return __call_result_t<get_domain_t, _Env&>();
|
||||
}
|
||||
else
|
||||
{
|
||||
return default_domain{};
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Env>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto
|
||||
basic_run_loop<_Env>::__attrs_t::query(get_completion_domain_t<set_stopped_t>) const noexcept
|
||||
{
|
||||
return query(get_completion_domain<set_value_t>);
|
||||
}
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_RUN_LOOP
|
||||
@@ -1,104 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_SCHEDULE_FROM
|
||||
#define __CUDAX_EXECUTION_SCHEDULE_FROM
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no sys
|
||||
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__cccl/unreachable.h>
|
||||
#include <cuda/std/__type_traits/conditional.h>
|
||||
#include <cuda/std/__utility/pod_tuple.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/concepts.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/exception.cuh>
|
||||
#include <cuda/experimental/__execution/get_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/meta.cuh>
|
||||
#include <cuda/experimental/__execution/queries.cuh>
|
||||
#include <cuda/experimental/__execution/rcvr_ref.cuh>
|
||||
#include <cuda/experimental/__execution/transform_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/transform_sender.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__execution/variant.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
struct schedule_from_t
|
||||
{
|
||||
template <class _Sndr>
|
||||
struct __sndr_t;
|
||||
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr __sndr) const noexcept
|
||||
{
|
||||
return __sndr_t<_Sndr>{{}, {}, _CCCL_MOVE(__sndr)};
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Sndr>
|
||||
struct schedule_from_t::__sndr_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class _Self, class... _Env>
|
||||
_CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
return get_child_completion_signatures<_Self, _Sndr, _Env...>();
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) && -> connect_result_t<_Sndr, _Rcvr>
|
||||
{
|
||||
return execution::connect(_CCCL_MOVE(__sndr_), _CCCL_MOVE(__rcvr));
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
connect(_Rcvr __rcvr) const& -> connect_result_t<const _Sndr&, _Rcvr>
|
||||
{
|
||||
return execution::connect(__sndr_, _CCCL_MOVE(__rcvr));
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __fwd_env_t<env_of_t<_Sndr>>
|
||||
{
|
||||
return __fwd_env(execution::get_env(__sndr_));
|
||||
}
|
||||
|
||||
schedule_from_t __tag{};
|
||||
::cuda::std::__ignore_t __ignore_;
|
||||
_Sndr __sndr_;
|
||||
};
|
||||
|
||||
template <class _Sndr>
|
||||
inline constexpr int structured_binding_size<schedule_from_t::__sndr_t<_Sndr>> = 3;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT schedule_from_t schedule_from{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_SCHEDULE_FROM
|
||||
@@ -1,301 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_SEQUENCE
|
||||
#define __CUDAX_EXECUTION_SEQUENCE
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__cccl/unreachable.h>
|
||||
#include <cuda/std/__tuple_dir/ignore.h>
|
||||
#include <cuda/std/__type_traits/conditional.h>
|
||||
#include <cuda/std/__type_traits/is_callable.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/exception.cuh>
|
||||
#include <cuda/experimental/__execution/get_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/rcvr_ref.cuh>
|
||||
#include <cuda/experimental/__execution/rcvr_with_env.cuh>
|
||||
#include <cuda/experimental/__execution/transform_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/transform_sender.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__execution/variant.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
namespace __detail
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Attrs, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
__mk_seq_env_next(const _Attrs& __attrs, const _Env&... __env) noexcept
|
||||
{
|
||||
if constexpr (__callable<get_completion_scheduler_t<set_value_t>, const _Attrs&, const _Env&...>)
|
||||
{
|
||||
return __mk_sch_env(get_completion_scheduler<set_value_t>(__attrs, __env...), __env...);
|
||||
}
|
||||
else if constexpr (__callable<get_completion_domain_t<set_value_t>, const _Attrs&, const _Env&...>)
|
||||
{
|
||||
using __domain_t = __call_result_t<get_completion_domain_t<set_value_t>, const _Attrs&, const _Env&...>;
|
||||
return prop{get_domain, __domain_t{}};
|
||||
}
|
||||
else
|
||||
{
|
||||
return env{};
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Attrs, class... _Env>
|
||||
using __seq_env_next_t = decltype(__detail::__mk_seq_env_next(declval<_Attrs>(), declval<_Env>()...));
|
||||
|
||||
//! @brief Given a completion tag type, an environment, and a pack of attributes objects
|
||||
//! obtained from a sequence of senders, return the scheduler on which the final sender
|
||||
//! would complete assuming each sender was started where the previous sender completed.
|
||||
// template <class _Tag, class _Env, class _Attrs>
|
||||
// [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __seq_compl_sch_for(const _Env& __env, const _Attrs& __attrs)
|
||||
// noexcept
|
||||
// {
|
||||
// return __call_or(get_completion_scheduler<_Tag>, __nil{}, __attrs, __env);
|
||||
// }
|
||||
|
||||
// template <class _Tag, class _Env, class _Attrs0, class _Attrs1, class... _Attrs>
|
||||
// [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __seq_compl_sch_for(
|
||||
// const _Env& __env, const _Attrs0& __attrs0, const _Attrs1& __attrs1, const _Attrs&... __attrs) noexcept
|
||||
// {
|
||||
// return __seq_compl_sch_for<_Tag>(__detail::__mk_seq_env_next(__attrs0, __env), __attrs1, __attrs...);
|
||||
// if constexpr (__callable<get_completion_scheduler_t<set_value_t>, const _Attrs0&, const _Env&>)
|
||||
// {
|
||||
// return;
|
||||
// }
|
||||
// auto __env_next = __detail::__mk_seq_env_next(__attrs0, __env);
|
||||
// return;
|
||||
// }
|
||||
} // namespace __detail
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT sequence_t
|
||||
{
|
||||
_CUDAX_SEMI_PRIVATE :
|
||||
template <class _Attrs, class... _Env>
|
||||
using __env2_t = __join_env_t<__detail::__seq_env_next_t<_Attrs, __fwd_env_t<_Env>...>, __fwd_env_t<_Env>...>;
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Attrs, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto
|
||||
__mk_env2(const _Attrs& __attrs, const _Env&... __env) noexcept -> __env2_t<_Attrs, _Env...>
|
||||
{
|
||||
return __join_env(__detail::__mk_seq_env_next(__attrs, __fwd_env(__env)...), __fwd_env(__env)...);
|
||||
}
|
||||
|
||||
template <class _Rcvr, class _Env2, class _Sndr2>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __state_t
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __state_t(_Rcvr&& __rcvr, _Env2 __env, _Sndr2&& __sndr2)
|
||||
: __rcvr2_{static_cast<_Rcvr&&>(__rcvr), __env}
|
||||
, __opstate2_(execution::connect(static_cast<_Sndr2&&>(__sndr2), __ref_rcvr(__rcvr2_)))
|
||||
{}
|
||||
|
||||
__rcvr_with_env_t<_Rcvr, _Env2> __rcvr2_;
|
||||
connect_result_t<_Sndr2, __rcvr_ref_t<__rcvr_with_env_t<_Rcvr, _Env2>>> __opstate2_;
|
||||
};
|
||||
|
||||
template <class _Rcvr, class _Env2, class _Sndr2>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_t
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
template <class... _Values>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_value(_Values&&...) noexcept
|
||||
{
|
||||
execution::start(__state_->__opstate2_);
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_error(_Error&& __error) noexcept
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(__state_->__rcvr2_.__base()), static_cast<_Error&&>(__error));
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_stopped() noexcept
|
||||
{
|
||||
execution::set_stopped(static_cast<_Rcvr&&>(__state_->__rcvr2_.__base()));
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __fwd_env_t<env_of_t<_Rcvr>>
|
||||
{
|
||||
return __fwd_env(execution::get_env(__state_->__rcvr2_.__base()));
|
||||
}
|
||||
|
||||
__state_t<_Rcvr, _Env2, _Sndr2>* __state_;
|
||||
};
|
||||
|
||||
template <class _Rcvr, class _Sndr1, class _Sndr2>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
using __env2_t _CCCL_NODEBUG_ALIAS = __detail::__seq_env_next_t<env_of_t<_Sndr1>, env_of_t<_Rcvr>>;
|
||||
|
||||
// The moves from lvalues here is intentional:
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_HOST_DEVICE_API constexpr __opstate_t(_Sndr1& __sndr1, _Sndr2& __sndr2, _Rcvr& __rcvr, __env2_t __env2)
|
||||
: __state_(static_cast<_Rcvr&&>(__rcvr), static_cast<__env2_t&&>(__env2), static_cast<_Sndr2&&>(__sndr2))
|
||||
, __opstate1_(execution::connect(static_cast<_Sndr1&&>(__sndr1), __rcvr_t<_Rcvr, __env2_t, _Sndr2>{&__state_}))
|
||||
{}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_HOST_DEVICE_API constexpr __opstate_t(_Sndr1&& __sndr1, _Sndr2&& __sndr2, _Rcvr&& __rcvr)
|
||||
: __opstate_t(__sndr1, __sndr2, __rcvr, __detail::__mk_seq_env_next(get_env(__sndr1), get_env(__rcvr)))
|
||||
{}
|
||||
|
||||
_CCCL_IMMOVABLE(__opstate_t);
|
||||
|
||||
_CCCL_HOST_DEVICE_API ~__opstate_t() {}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void start() noexcept
|
||||
{
|
||||
execution::start(__opstate1_);
|
||||
}
|
||||
|
||||
private:
|
||||
__state_t<_Rcvr, __env2_t, _Sndr2> __state_;
|
||||
connect_result_t<_Sndr1, __rcvr_t<_Rcvr, __env2_t, _Sndr2>> __opstate1_;
|
||||
};
|
||||
|
||||
public:
|
||||
template <class _Sndr1, class _Sndr2>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sndr1, class _Sndr2>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr1 __sndr1, _Sndr2 __sndr2) const;
|
||||
};
|
||||
|
||||
template <class _Sndr1, class _Sndr2>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT sequence_t::__sndr_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
template <class... _Env>
|
||||
using __env2_t _CCCL_NODEBUG_ALIAS = sequence_t::__env2_t<env_of_t<_Sndr1>, _Env...>;
|
||||
|
||||
template <class _Self, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
_CUDAX_LET_COMPLETIONS(auto(__completions1) = get_child_completion_signatures<_Self, _Sndr1, _Env...>())
|
||||
{
|
||||
_CUDAX_LET_COMPLETIONS(auto(__completions2) = get_child_completion_signatures<_Self, _Sndr2, __env2_t<_Env...>>())
|
||||
{
|
||||
// __swallow_transform to ignore the first sender's value completions
|
||||
return __completions2 + transform_completion_signatures(__completions1, __swallow_transform{});
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) && //
|
||||
-> sequence_t::__opstate_t<_Rcvr, _Sndr1, _Sndr2>
|
||||
{
|
||||
using __opstate_t = sequence_t::__opstate_t<_Rcvr, _Sndr1, _Sndr2>;
|
||||
return __opstate_t{static_cast<_Sndr1&&>(__sndr1_), static_cast<_Sndr2>(__sndr2_), static_cast<_Rcvr&&>(__rcvr)};
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) const& //
|
||||
-> sequence_t::__opstate_t<_Rcvr, const _Sndr1&, const _Sndr2&>
|
||||
{
|
||||
using __opstate_t = sequence_t::__opstate_t<_Rcvr, const _Sndr1&, const _Sndr2&>;
|
||||
return __opstate_t{__sndr1_, __sndr2_, static_cast<_Rcvr&&>(__rcvr)};
|
||||
}
|
||||
|
||||
struct __attrs_t
|
||||
{
|
||||
// If _Sndr2 has _SetTag completions but does not know its _SetTag completion scheduler,
|
||||
// then we cannot know it either. Delete the function to prevent its use.
|
||||
_CCCL_TEMPLATE(class _SetTag, class... _Env)
|
||||
_CCCL_REQUIRES(__has_completions_for<_Sndr2, _SetTag, __env2_t<_Env...>> _CCCL_AND(
|
||||
!__callable<get_completion_scheduler_t<_SetTag>, env_of_t<_Sndr2>, __env2_t<_Env...>>))
|
||||
_CCCL_HOST_DEVICE_API auto query(get_completion_scheduler_t<_SetTag>, const _Env&...) const = delete;
|
||||
|
||||
// If _Sndr2 has _SetTag completions but does not know its _SetTag completion domain,
|
||||
// then we cannot know it either. Delete the function to prevent its use.
|
||||
_CCCL_TEMPLATE(class _SetTag, class... _Env)
|
||||
_CCCL_REQUIRES(__has_completions_for<_Sndr2, _SetTag, __env2_t<_Env...>> _CCCL_AND(
|
||||
!__callable<get_completion_domain_t<_SetTag>, env_of_t<_Sndr2>, __env2_t<_Env...>>))
|
||||
_CCCL_HOST_DEVICE_API auto query(get_completion_domain_t<_SetTag>, const _Env&...) const = delete;
|
||||
|
||||
template <class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_behavior_t, const _Env&...) const noexcept
|
||||
{
|
||||
return (execution::min) (execution::get_completion_behavior<_Sndr1, __fwd_env_t<_Env>...>(),
|
||||
execution::get_completion_behavior<_Sndr2, __env2_t<_Env...>>());
|
||||
}
|
||||
|
||||
using __child_attrs_t = __join_env_t<env_of_t<_Sndr2>, env_of_t<_Sndr1>>;
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Query, class... _Args)
|
||||
_CCCL_REQUIRES(__forwarding_query<_Query> _CCCL_AND __queryable_with<__child_attrs_t, _Query, _Args...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(_Query, _Args&&... __args) const
|
||||
noexcept(__nothrow_queryable_with<__child_attrs_t, _Query, _Args...>)
|
||||
-> __query_result_t<__child_attrs_t, _Query, _Args...>
|
||||
{
|
||||
auto&& __env = __join_env(execution::get_env(__self_->__sndr2_), execution::get_env(__self_->__sndr1_));
|
||||
return __env.query(_Query{}, static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
__sndr_t const* __self_;
|
||||
};
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __attrs_t
|
||||
{
|
||||
return {this};
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ sequence_t __tag_;
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ ::cuda::std::__ignore_t __ign_;
|
||||
_Sndr1 __sndr1_;
|
||||
_Sndr2 __sndr2_;
|
||||
};
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sndr1, class _Sndr2>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto sequence_t::operator()(_Sndr1 __sndr1, _Sndr2 __sndr2) const
|
||||
{
|
||||
using __sndr_t _CCCL_NODEBUG_ALIAS = sequence_t::__sndr_t<_Sndr1, _Sndr2>;
|
||||
return __sndr_t{{}, {}, static_cast<_Sndr1&&>(__sndr1), static_cast<_Sndr2&&>(__sndr2)};
|
||||
}
|
||||
|
||||
template <class _Sndr1, class _Sndr2>
|
||||
inline constexpr int structured_binding_size<sequence_t::__sndr_t<_Sndr1, _Sndr2>> = 4;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT sequence_t sequence{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_SEQUENCE
|
||||
@@ -1,68 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_SNDR_REF
|
||||
#define __CUDAX_EXECUTION_SNDR_REF
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/get_completion_signatures.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
template <class _Sndr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_ref
|
||||
{
|
||||
using sender_concept = receiver_t;
|
||||
|
||||
_CCCL_HOST_DEVICE_API explicit constexpr __sndr_ref(_Sndr&& __sndr) noexcept
|
||||
: __sndr_(static_cast<_Sndr&&>(__sndr))
|
||||
{}
|
||||
|
||||
template <class _Self, class... _Env>
|
||||
_CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
return execution::get_completion_signatures<_Sndr, _Env...>();
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) const
|
||||
{
|
||||
return execution::connect(static_cast<_Sndr&&>(__sndr_), static_cast<_Rcvr&&>(__rcvr));
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> env_of_t<_Sndr>
|
||||
{
|
||||
return execution::get_env(__sndr_);
|
||||
}
|
||||
|
||||
private:
|
||||
_Sndr&& __sndr_;
|
||||
};
|
||||
|
||||
template <class _Sndr>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES __sndr_ref(_Sndr&& __sndr) -> __sndr_ref<_Sndr>;
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_SNDR_REF
|
||||
@@ -1,112 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_START_DETACHED
|
||||
#define __CUDAX_EXECUTION_START_DETACHED
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__exception/terminate.h>
|
||||
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/apply_sender.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
struct start_detached_t
|
||||
{
|
||||
private:
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_base_t
|
||||
{};
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_t
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
__opstate_base_t* __opstate_;
|
||||
void (*__destroy)(__opstate_base_t*) noexcept;
|
||||
|
||||
template <class... _As>
|
||||
constexpr void set_value(_As&&...) noexcept
|
||||
{
|
||||
__destroy(__opstate_);
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
constexpr void set_error(_Error&&) noexcept
|
||||
{
|
||||
::cuda::std::terminate();
|
||||
}
|
||||
|
||||
constexpr void set_stopped() noexcept
|
||||
{
|
||||
__destroy(__opstate_);
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Sndr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t : __opstate_base_t
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
connect_result_t<_Sndr, __rcvr_t> __opstate_;
|
||||
|
||||
static void __destroy(__opstate_base_t* __ptr) noexcept
|
||||
{
|
||||
delete static_cast<__opstate_t*>(__ptr);
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __opstate_t(_Sndr&& __sndr)
|
||||
: __opstate_(execution::connect(static_cast<_Sndr&&>(__sndr), __rcvr_t{this, &__destroy}))
|
||||
{}
|
||||
|
||||
_CCCL_IMMOVABLE(__opstate_t);
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void start() noexcept
|
||||
{
|
||||
execution::start(__opstate_);
|
||||
}
|
||||
};
|
||||
|
||||
public:
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API static auto apply_sender(_Sndr __sndr)
|
||||
{
|
||||
execution::start(*new __opstate_t<_Sndr>{static_cast<_Sndr&&>(__sndr)});
|
||||
}
|
||||
|
||||
/// run detached.
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API void operator()(_Sndr __sndr) const
|
||||
{
|
||||
using __domain_t _CCCL_NODEBUG_ALIAS = __completion_domain_of_t<set_value_t, _Sndr, env<>>;
|
||||
execution::apply_sender(__domain_t{}, *this, static_cast<_Sndr&&>(__sndr));
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT start_detached_t start_detached{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_START_DETACHED
|
||||
@@ -1,212 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_STARTS_ON
|
||||
#define __CUDAX_EXECUTION_STARTS_ON
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__cccl/unreachable.h>
|
||||
#include <cuda/std/__tuple_dir/ignore.h>
|
||||
#include <cuda/std/__utility/forward_like.h>
|
||||
|
||||
#include <cuda/experimental/__execution/continues_on.cuh>
|
||||
#include <cuda/experimental/__execution/get_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/just.cuh>
|
||||
#include <cuda/experimental/__execution/sequence.cuh>
|
||||
#include <cuda/experimental/__execution/transform_sender.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
template <class _Query>
|
||||
_CCCL_CONCEPT __forwarding_starts_on_query = __forwarding_query<_Query> && !__is_completion_query<_Query>;
|
||||
|
||||
//! @brief Execution algorithm that starts a given sender on a specified scheduler.
|
||||
//!
|
||||
//! The `starts_on` algorithm takes a scheduler and a sender, and returns a new sender
|
||||
//! that, when connected and started, will first schedule work on the provided scheduler,
|
||||
//! and then start the original sender on that scheduler's execution context.
|
||||
//!
|
||||
//! This algorithm is particularly useful for ensuring that a chain of work begins
|
||||
//! execution on a specific execution context, such as a particular GPU stream or thread
|
||||
//! pool.
|
||||
//!
|
||||
//! @details The operation proceeds in two phases:
|
||||
//! 1. **Scheduling Phase**: The algorithm first calls `schedule()` on the provided
|
||||
//! scheduler to obtain a sender that represents scheduling work on that scheduler's
|
||||
//! execution context.
|
||||
//! 2. **Execution Phase**: Once the scheduling operation completes successfully, the
|
||||
//! original sender is started on the scheduler's execution context.
|
||||
//!
|
||||
//! The resulting sender's completion signatures are derived from both the scheduler's
|
||||
//! `schedule()` sender and the original sender. Error and stopped signals from either
|
||||
//! operation are propagated to the final receiver.
|
||||
//!
|
||||
//! @tparam _Sch A scheduler type that satisfies the `scheduler` concept
|
||||
//! @tparam _Sndr A sender type that satisfies the `sender` concept
|
||||
//!
|
||||
//! @param __sch The scheduler on which the sender should start execution
|
||||
//! @param __sndr The sender to be started on the scheduler's execution context
|
||||
//!
|
||||
//! @return A sender that, when started, will first schedule on `__sch` and then execute
|
||||
//! `__sndr`
|
||||
//!
|
||||
//! @note The returned sender's environment includes the provided scheduler as the current
|
||||
//! scheduler, allowing nested senders to query and use the same execution context.
|
||||
//!
|
||||
//! @note This implementation follows the C++26 standard specification for
|
||||
//! `std::execution::starts_on` as defined in [exec.starts.on].
|
||||
//!
|
||||
//! Example usage:
|
||||
//! @code
|
||||
//! auto work = cuda::experimental::execution::just(42)
|
||||
//! | cuda::experimental::execution::then([](int x) { return x * 2; });
|
||||
//!
|
||||
//! auto scheduled_work = cuda::experimental::execution::starts_on(some_scheduler, work);
|
||||
//! @endcode
|
||||
//!
|
||||
//! @see schedule
|
||||
//! @see scheduler
|
||||
//! @see sender
|
||||
//! @see receiver
|
||||
struct starts_on_t
|
||||
{
|
||||
template <class _Sch, class _Sndr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
private:
|
||||
template <class _Sch, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto __mk_env2(_Sch __sch, _Env&&... __env)
|
||||
{
|
||||
return __join_env(__mk_sch_env(__sch, __env...), __fwd_env(static_cast<_Env&&>(__env))...);
|
||||
}
|
||||
|
||||
template <class _Sch, class... _Env>
|
||||
using __env2_t = decltype(__mk_env2(declval<_Sch>(), declval<_Env>()...));
|
||||
|
||||
template <class _Sch, class _Sndr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __attrs_t
|
||||
{
|
||||
// If the sender has a _SetTag completion, then the completion scheduler for _SetTag
|
||||
// is the sender's.
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _SetTag, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_scheduler_t<_SetTag>, _Env&&... __env) const noexcept
|
||||
-> __call_result_t<get_completion_scheduler_t<_SetTag>, env_of_t<_Sndr>, __env2_t<_Sch, _Env>...>
|
||||
{
|
||||
return get_completion_scheduler<_SetTag>(
|
||||
execution::get_env(__self_->__sndr_), __mk_env2(__self_->__sch_, static_cast<_Env&&>(__env))...);
|
||||
}
|
||||
|
||||
// If the sender has a _SetTag completion, then the completion scheduler for _SetTag
|
||||
// is the sender's.
|
||||
template <class _SetTag, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_domain_t<_SetTag>, _Env&&... __env) const noexcept
|
||||
-> __call_result_t<get_completion_domain_t<_SetTag>, env_of_t<_Sndr>, __env2_t<_Sch, _Env>...>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
template <class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_behavior_t, _Env&&...) const noexcept
|
||||
{
|
||||
return (execution::min) (execution::get_completion_behavior<schedule_result_t<_Sch>, __fwd_env_t<_Env>...>(),
|
||||
execution::get_completion_behavior<_Sndr, __env2_t<_Sch, _Env>...>());
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Query, class... _Args)
|
||||
_CCCL_REQUIRES(__forwarding_starts_on_query<_Query> _CCCL_AND __queryable_with<env_of_t<_Sndr>, _Query, _Args...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(_Query, _Args&&... __args) const
|
||||
noexcept(__nothrow_queryable_with<env_of_t<_Sndr>, _Query, _Args...>)
|
||||
-> __query_result_t<env_of_t<_Sndr>, _Query, _Args...>
|
||||
{
|
||||
return execution::get_env(__self_->__sndr_).query(_Query{}, static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
const __sndr_t<_Sch, _Sndr>* __self_;
|
||||
};
|
||||
|
||||
public:
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] static _CCCL_HOST_DEVICE_API constexpr auto
|
||||
transform_sender(start_t, _Sndr&& __sndr, ::cuda::std::__ignore_t)
|
||||
{
|
||||
auto&& [__ign, __sch, __child] = __sndr;
|
||||
return sequence(continues_on(just(), __sch), ::cuda::std::forward_like<_Sndr>(__child));
|
||||
}
|
||||
|
||||
template <class _Sch, class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Sch __sch, _Sndr __sndr) const;
|
||||
};
|
||||
|
||||
template <class _Sch, class _Sndr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT starts_on_t::__sndr_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class _Self, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
_CUDAX_LET_COMPLETIONS(
|
||||
auto(__child_completions) = execution::get_child_completion_signatures<_Self, _Sndr, __env2_t<_Sch, _Env>...>())
|
||||
{
|
||||
_CUDAX_LET_COMPLETIONS(
|
||||
auto(__sch_completions) = execution::get_completion_signatures<schedule_result_t<_Sch>, __fwd_env_t<_Env>...>())
|
||||
{
|
||||
// The scheduler contributes error and stopped completions.
|
||||
auto __sch_err_stop_completions = transform_completion_signatures(__sch_completions, __swallow_transform{});
|
||||
return __child_completions + __sch_err_stop_completions;
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __attrs_t<_Sch, _Sndr>
|
||||
{
|
||||
return __attrs_t<_Sch, _Sndr>{this};
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ starts_on_t __tag_;
|
||||
_Sch __sch_;
|
||||
_Sndr __sndr_;
|
||||
};
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Sch, class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto starts_on_t::operator()(_Sch __sch, _Sndr __sndr) const
|
||||
{
|
||||
static_assert(__is_scheduler<_Sch>, "starts_on requires a scheduler as the first argument");
|
||||
static_assert(__is_sender<_Sndr>, "starts_on requires a sender as the second argument");
|
||||
return __sndr_t<_Sch, _Sndr>{{}, static_cast<_Sch&&>(__sch), static_cast<_Sndr&&>(__sndr)};
|
||||
}
|
||||
|
||||
template <class _Sch, class _Sndr>
|
||||
inline constexpr int structured_binding_size<starts_on_t::__sndr_t<_Sch, _Sndr>> = 3;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT starts_on_t starts_on{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_STARTS_ON
|
||||
@@ -1,518 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_STOP_TOKEN
|
||||
#define __CUDAX_EXECUTION_STOP_TOKEN
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__concepts/boolean_testable.h>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__concepts/equality_comparable.h>
|
||||
#include <cuda/std/__thread/threading_support.h>
|
||||
#include <cuda/std/__tuple_dir/ignore.h>
|
||||
#include <cuda/std/__type_traits/is_nothrow_constructible.h>
|
||||
#include <cuda/std/atomic>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/thread.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
|
||||
#if __has_include(<stop_token>) && __cpp_lib_jthread >= 201911
|
||||
# include <stop_token>
|
||||
#endif // __has_include(<stop_token>) && __cpp_lib_jthread >= 201911
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
// [stoptoken.inplace], class inplace_stop_token
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT inplace_stop_token;
|
||||
|
||||
// [stopsource.inplace], class inplace_stop_source
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT inplace_stop_source;
|
||||
|
||||
// [stopcallback.inplace], class template inplace_stop_callback
|
||||
template <class _Callback>
|
||||
class inplace_stop_callback;
|
||||
|
||||
namespace __stok
|
||||
{
|
||||
struct __inplace_stop_callback_base
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API constexpr void __execute() noexcept
|
||||
{
|
||||
this->__execute_fn_(this);
|
||||
}
|
||||
|
||||
protected:
|
||||
using __execute_fn_t _CCCL_NODEBUG_ALIAS = void(__inplace_stop_callback_base*) noexcept;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __inplace_stop_callback_base(
|
||||
const inplace_stop_source* __source, __execute_fn_t* __execute) noexcept
|
||||
: __source_(__source)
|
||||
, __execute_fn_(__execute)
|
||||
{}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void __register_callback() noexcept;
|
||||
|
||||
friend inplace_stop_source;
|
||||
|
||||
const inplace_stop_source* __source_;
|
||||
__execute_fn_t* __execute_fn_;
|
||||
__inplace_stop_callback_base* __next_ = nullptr;
|
||||
__inplace_stop_callback_base** __prev_ptr_ = nullptr;
|
||||
bool* __removed_during_callback_ = nullptr;
|
||||
::cuda::std::atomic<bool> __callback_completed_{false};
|
||||
};
|
||||
|
||||
struct __spin_wait
|
||||
{
|
||||
_CCCL_HIDE_FROM_ABI __spin_wait() noexcept = default;
|
||||
|
||||
_CCCL_HOST_DEVICE_API void __wait() noexcept
|
||||
{
|
||||
if (__count_ == 0)
|
||||
{
|
||||
execution::__this_thread_yield();
|
||||
}
|
||||
else
|
||||
{
|
||||
--__count_;
|
||||
::cuda::std::__cccl_thread_yield_processor();
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
static constexpr uint32_t __yield_threshold = 20;
|
||||
uint32_t __count_ = __yield_threshold;
|
||||
};
|
||||
|
||||
template <template <class> class>
|
||||
struct __check_type_alias_exists;
|
||||
} // namespace __stok
|
||||
|
||||
// [stoptoken.never], class never_stop_token
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT never_stop_token
|
||||
{
|
||||
private:
|
||||
struct __callback_type
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __callback_type(never_stop_token, ::cuda::std::__ignore_t) noexcept {}
|
||||
};
|
||||
|
||||
public:
|
||||
template <class>
|
||||
using callback_type _CCCL_NODEBUG_ALIAS = __callback_type;
|
||||
|
||||
_CCCL_HOST_DEVICE_API static constexpr auto stop_requested() noexcept -> bool
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API static constexpr auto stop_possible() noexcept -> bool
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API friend constexpr auto operator==(const never_stop_token&, const never_stop_token&) noexcept
|
||||
-> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API friend constexpr auto operator!=(const never_stop_token&, const never_stop_token&) noexcept
|
||||
-> bool
|
||||
{
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Callback>
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT inplace_stop_callback;
|
||||
|
||||
// [stopsource.inplace], class inplace_stop_source
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT inplace_stop_source
|
||||
{
|
||||
public:
|
||||
_CCCL_HIDE_FROM_ABI inplace_stop_source() noexcept = default;
|
||||
_CCCL_HOST_DEVICE_API ~inplace_stop_source();
|
||||
inplace_stop_source(inplace_stop_source&&) = delete;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr auto get_token() const noexcept -> inplace_stop_token;
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto request_stop() noexcept -> bool;
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto stop_requested() const noexcept -> bool
|
||||
{
|
||||
return (__state_.load(::cuda::std::memory_order_acquire) & __stop_requested_flag) != 0;
|
||||
}
|
||||
|
||||
private:
|
||||
friend inplace_stop_token;
|
||||
friend __stok::__inplace_stop_callback_base;
|
||||
template <class>
|
||||
friend class inplace_stop_callback;
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto __lock() const noexcept -> uint8_t;
|
||||
_CCCL_HOST_DEVICE_API void __unlock(uint8_t) const noexcept;
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto __try_lock_unless_stop_requested(bool) const noexcept -> bool;
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto __try_add_callback(__stok::__inplace_stop_callback_base*) const noexcept -> bool;
|
||||
|
||||
_CCCL_HOST_DEVICE_API void __remove_callback(__stok::__inplace_stop_callback_base*) const noexcept;
|
||||
|
||||
static constexpr uint8_t __stop_requested_flag = 1;
|
||||
static constexpr uint8_t __locked_flag = 2;
|
||||
|
||||
mutable ::cuda::std::atomic<uint8_t> __state_{0};
|
||||
mutable __stok::__inplace_stop_callback_base* __callbacks_ = nullptr;
|
||||
execution::__thread_id __notifying_thread_;
|
||||
};
|
||||
|
||||
// [stoptoken.inplace], class inplace_stop_token
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT inplace_stop_token
|
||||
{
|
||||
public:
|
||||
template <class _Fun>
|
||||
using callback_type _CCCL_NODEBUG_ALIAS = inplace_stop_callback<_Fun>;
|
||||
|
||||
_CCCL_HIDE_FROM_ABI inplace_stop_token() = default;
|
||||
|
||||
_CCCL_HIDE_FROM_ABI inplace_stop_token(const inplace_stop_token& __other) noexcept = default;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr inplace_stop_token(inplace_stop_token&& __other) noexcept
|
||||
: __source_(execution::__exchange(__other.__source_, {}))
|
||||
{}
|
||||
|
||||
_CCCL_HIDE_FROM_ABI constexpr auto operator=(const inplace_stop_token& __other) noexcept
|
||||
-> inplace_stop_token& = default;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator=(inplace_stop_token&& __other) noexcept -> inplace_stop_token&
|
||||
{
|
||||
__source_ = execution::__exchange(__other.__source_, nullptr);
|
||||
return *this;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto stop_requested() const noexcept -> bool
|
||||
{
|
||||
return __source_ != nullptr && __source_->stop_requested();
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto stop_possible() const noexcept -> bool
|
||||
{
|
||||
return __source_ != nullptr;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void swap(inplace_stop_token& __other) noexcept
|
||||
{
|
||||
execution::__swap(__source_, __other.__source_);
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API friend constexpr auto
|
||||
operator==(const inplace_stop_token& __a, const inplace_stop_token& __b) noexcept -> bool
|
||||
{
|
||||
return __a.__source_ == __b.__source_;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API friend constexpr auto
|
||||
operator!=(const inplace_stop_token& __a, const inplace_stop_token& __b) noexcept -> bool
|
||||
{
|
||||
return __a.__source_ != __b.__source_;
|
||||
}
|
||||
|
||||
private:
|
||||
friend inplace_stop_source;
|
||||
template <class>
|
||||
friend class inplace_stop_callback;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit inplace_stop_token(const inplace_stop_source* __source) noexcept
|
||||
: __source_(__source)
|
||||
{}
|
||||
|
||||
const inplace_stop_source* __source_ = nullptr;
|
||||
};
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr auto inplace_stop_source::get_token() const noexcept -> inplace_stop_token
|
||||
{
|
||||
return inplace_stop_token{this};
|
||||
}
|
||||
|
||||
// [stopcallback.inplace], class template inplace_stop_callback
|
||||
template <class _Fun>
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT inplace_stop_callback : __stok::__inplace_stop_callback_base
|
||||
{
|
||||
public:
|
||||
template <class _Fun2>
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit inplace_stop_callback(inplace_stop_token __token, _Fun2&& __fun) noexcept(
|
||||
__nothrow_constructible<_Fun, _Fun2>)
|
||||
: __stok::__inplace_stop_callback_base(__token.__source_, &inplace_stop_callback::__execute_impl)
|
||||
, __fun(static_cast<_Fun2&&>(__fun))
|
||||
{
|
||||
__register_callback();
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API ~inplace_stop_callback()
|
||||
{
|
||||
if (__source_ != nullptr)
|
||||
{
|
||||
__source_->__remove_callback(this);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
_CCCL_HOST_DEVICE_API static constexpr void __execute_impl(__stok::__inplace_stop_callback_base* __cb) noexcept
|
||||
{
|
||||
static_cast<_Fun&&>(static_cast<inplace_stop_callback*>(__cb)->__fun)();
|
||||
}
|
||||
|
||||
_Fun __fun;
|
||||
};
|
||||
|
||||
namespace __stok
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API constexpr void __inplace_stop_callback_base::__register_callback() noexcept
|
||||
{
|
||||
if (__source_ != nullptr)
|
||||
{
|
||||
if (!__source_->__try_add_callback(this))
|
||||
{
|
||||
__source_ = nullptr;
|
||||
// _Callback not registered because stop_requested() was true.
|
||||
// Execute inline here.
|
||||
__execute();
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace __stok
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline inplace_stop_source::~inplace_stop_source()
|
||||
{
|
||||
_CCCL_ASSERT((__state_.load(::cuda::std::memory_order_relaxed) & __locked_flag) == 0, "");
|
||||
_CCCL_ASSERT(__callbacks_ == nullptr, "");
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline auto inplace_stop_source::request_stop() noexcept -> bool
|
||||
{
|
||||
if (!__try_lock_unless_stop_requested(true))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
__notifying_thread_ = execution::__this_thread_id();
|
||||
|
||||
// We are responsible for executing callbacks.
|
||||
while (__callbacks_ != nullptr)
|
||||
{
|
||||
auto* __callbk = __callbacks_;
|
||||
__callbk->__prev_ptr_ = nullptr;
|
||||
__callbacks_ = __callbk->__next_;
|
||||
if (__callbacks_ != nullptr)
|
||||
{
|
||||
__callbacks_->__prev_ptr_ = &__callbacks_;
|
||||
}
|
||||
|
||||
__state_.store(__stop_requested_flag, ::cuda::std::memory_order_release);
|
||||
|
||||
bool __removed_during_callback_ = false;
|
||||
__callbk->__removed_during_callback_ = &__removed_during_callback_;
|
||||
|
||||
__callbk->__execute();
|
||||
|
||||
if (!__removed_during_callback_)
|
||||
{
|
||||
__callbk->__removed_during_callback_ = nullptr;
|
||||
__callbk->__callback_completed_.store(true, ::cuda::std::memory_order_release);
|
||||
}
|
||||
|
||||
__lock();
|
||||
}
|
||||
|
||||
__state_.store(__stop_requested_flag, ::cuda::std::memory_order_release);
|
||||
return false;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline auto inplace_stop_source::__lock() const noexcept -> uint8_t
|
||||
{
|
||||
__stok::__spin_wait __spin;
|
||||
auto __old_state = __state_.load(::cuda::std::memory_order_relaxed);
|
||||
do
|
||||
{
|
||||
while ((__old_state & __locked_flag) != 0)
|
||||
{
|
||||
__spin.__wait();
|
||||
__old_state = __state_.load(::cuda::std::memory_order_relaxed);
|
||||
}
|
||||
} while (!__state_.compare_exchange_weak(
|
||||
__old_state, __old_state | __locked_flag, ::cuda::std::memory_order_acquire, ::cuda::std::memory_order_relaxed));
|
||||
|
||||
return __old_state;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline void inplace_stop_source::__unlock(uint8_t __old_state) const noexcept
|
||||
{
|
||||
(void) __state_.store(__old_state, ::cuda::std::memory_order_release);
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline auto
|
||||
inplace_stop_source::__try_lock_unless_stop_requested(bool __set_stop_requested) const noexcept -> bool
|
||||
{
|
||||
__stok::__spin_wait __spin;
|
||||
auto __old_state = __state_.load(::cuda::std::memory_order_relaxed);
|
||||
do
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
if ((__old_state & __stop_requested_flag) != 0)
|
||||
{
|
||||
// Stop already requested.
|
||||
return false;
|
||||
}
|
||||
else if (__old_state == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
__spin.__wait();
|
||||
__old_state = __state_.load(::cuda::std::memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
} while (!__state_.compare_exchange_weak(
|
||||
__old_state,
|
||||
__set_stop_requested ? (__locked_flag | __stop_requested_flag) : __locked_flag,
|
||||
::cuda::std::memory_order_acq_rel,
|
||||
::cuda::std::memory_order_relaxed));
|
||||
|
||||
// Lock acquired successfully
|
||||
return true;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline auto
|
||||
inplace_stop_source::__try_add_callback(__stok::__inplace_stop_callback_base* __callbk) const noexcept -> bool
|
||||
{
|
||||
if (!__try_lock_unless_stop_requested(false))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
__callbk->__next_ = __callbacks_;
|
||||
__callbk->__prev_ptr_ = &__callbacks_;
|
||||
if (__callbacks_ != nullptr)
|
||||
{
|
||||
__callbacks_->__prev_ptr_ = &__callbk->__next_;
|
||||
}
|
||||
__callbacks_ = __callbk;
|
||||
|
||||
__unlock(0);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline void
|
||||
inplace_stop_source::__remove_callback(__stok::__inplace_stop_callback_base* __callbk) const noexcept
|
||||
{
|
||||
auto __old_state = __lock();
|
||||
|
||||
if (__callbk->__prev_ptr_ != nullptr)
|
||||
{
|
||||
// _Callback has not been executed yet.
|
||||
// Remove from the list.
|
||||
*__callbk->__prev_ptr_ = __callbk->__next_;
|
||||
if (__callbk->__next_ != nullptr)
|
||||
{
|
||||
__callbk->__next_->__prev_ptr_ = __callbk->__prev_ptr_;
|
||||
}
|
||||
__unlock(__old_state);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto __notifying_thread_ = this->__notifying_thread_;
|
||||
__unlock(__old_state);
|
||||
|
||||
// _Callback has either already been executed or is
|
||||
// currently executing on another thread.
|
||||
if (execution::__this_thread_id() == __notifying_thread_)
|
||||
{
|
||||
if (__callbk->__removed_during_callback_ != nullptr)
|
||||
{
|
||||
*__callbk->__removed_during_callback_ = true;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Concurrently executing on another thread.
|
||||
// Wait until the other thread finishes executing the callback.
|
||||
__stok::__spin_wait __spin;
|
||||
while (!__callbk->__callback_completed_.load(::cuda::std::memory_order_acquire))
|
||||
{
|
||||
__spin.__wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct __on_stop_request
|
||||
{
|
||||
inplace_stop_source& __source_;
|
||||
|
||||
_CCCL_HOST_DEVICE_API void operator()() const noexcept
|
||||
{
|
||||
__source_.request_stop();
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Token, class _Callback>
|
||||
using stop_callback_for_t _CCCL_NODEBUG_ALIAS = typename _Token::template callback_type<_Callback>;
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
template <template <class> class>
|
||||
struct __check_type_alias_exists;
|
||||
} // namespace __detail
|
||||
|
||||
template <class _Token>
|
||||
_CCCL_CONCEPT stoppable_token = _CCCL_REQUIRES_EXPR((_Token), const _Token& __token)(
|
||||
requires(__nothrow_copyable<_Token>),
|
||||
requires(__nothrow_movable<_Token>),
|
||||
requires(cuda::std::equality_comparable<_Token>),
|
||||
_Satisfies(cuda::std::__boolean_testable) __token.stop_requested(),
|
||||
_Satisfies(cuda::std::__boolean_testable) __token.stop_possible(),
|
||||
noexcept(__token.stop_requested()),
|
||||
noexcept(__token.stop_possible()),
|
||||
typename(__detail::__check_type_alias_exists<_Token::template callback_type>));
|
||||
|
||||
template <class _Token, typename _Callback, typename _Initializer = _Callback>
|
||||
_CCCL_CONCEPT stoppable_token_for = _CCCL_REQUIRES_EXPR((_Token, _Callback, _Initializer))(
|
||||
requires(stoppable_token<_Token>),
|
||||
requires(__callable<_Callback>),
|
||||
typename(stop_callback_for_t<_Token, _Callback>),
|
||||
requires(__constructible<_Callback, _Initializer>),
|
||||
requires(__constructible<stop_callback_for_t<_Token, _Callback>, const _Token&, _Initializer>));
|
||||
|
||||
template <class _Token>
|
||||
_CCCL_CONCEPT unstoppable_token = _CCCL_REQUIRES_EXPR((_Token))(
|
||||
requires(stoppable_token<_Token>),
|
||||
_Satisfies(cuda::std::__boolean_testable) _Token::stop_possible(),
|
||||
requires(!_Token::stop_possible()));
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_STOP_TOKEN
|
||||
@@ -1,434 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_STREAM_ADAPTOR
|
||||
#define __CUDAX_EXECUTION_STREAM_ADAPTOR
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__launch/configuration.h>
|
||||
#include <cuda/__launch/launch.h>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__memory/unique_ptr.h>
|
||||
#include <cuda/std/__type_traits/remove_cvref.h>
|
||||
#include <cuda/std/__utility/pod_tuple.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/domain.cuh>
|
||||
#include <cuda/experimental/__execution/get_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/stream/domain.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__execution/variant.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
#include <cuda/experimental/__launch/launch.cuh>
|
||||
#include <cuda/experimental/__stream/stream_ref.cuh>
|
||||
|
||||
#include <nv/target>
|
||||
|
||||
#include <cuda_runtime_api.h>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
// This header provides a sender adaptor that adapts a non-stream sender to a stream
|
||||
// sender. The adaptor does several things:
|
||||
//
|
||||
// 1. It ensures that the stream_domain is used for sender transformations.
|
||||
// 2. It takes the launch configuration from the child sender and puts it into the
|
||||
// environment of the inner receiver used to connect the child sender.
|
||||
// 3. It connects the child sender to the inner receiver, which will write the child's
|
||||
// results into a variant that is in managed memory.
|
||||
// 4. It creates the child operation state in managed memory.
|
||||
// 5. It starts the child operation on the host, which causes the predecessor kernels to
|
||||
// be launched in order.
|
||||
// 6. It launches a completion kernel on the stream to read the results out of the variant
|
||||
// and send them to the outer receiver. The launch configuration is read from the outer
|
||||
// receiver's environment.
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
namespace __stream
|
||||
{
|
||||
struct __complete_rcvr
|
||||
{
|
||||
template <class _Rcvr, class _Tag, class... _Args>
|
||||
_CCCL_HOST_DEVICE_API void operator()(_Rcvr& __rcvr, _Tag, _Args&&... __args) const noexcept
|
||||
{
|
||||
_Tag{}(static_cast<_Rcvr&&>(__rcvr), static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
};
|
||||
|
||||
struct __visit_results
|
||||
{
|
||||
template <class _Rcvr, class _Tuple>
|
||||
_CCCL_HOST_DEVICE_API void operator()(_Rcvr& __rcvr, _Tuple&& __tuple) const noexcept
|
||||
{
|
||||
::cuda::std::__apply(__complete_rcvr{}, static_cast<_Tuple&&>(__tuple), __rcvr);
|
||||
}
|
||||
};
|
||||
|
||||
// __state_t lives in managed memory. It stores everything the operation state needs,
|
||||
// besides the child operation state.
|
||||
template <class _Rcvr, class _Variant>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __state_base_t
|
||||
{
|
||||
_Rcvr __rcvr_;
|
||||
_Variant __results_;
|
||||
bool __complete_inline_;
|
||||
};
|
||||
|
||||
template <class _Rcvr, class _Config, class _Variant>
|
||||
struct __state_t : __state_base_t<_Rcvr, _Variant>
|
||||
{
|
||||
_Config __launch_config_;
|
||||
};
|
||||
|
||||
// remove any exception_ptr error completion from the completion signatures, and replace it
|
||||
// with a cudaError_t error completion.
|
||||
template <class _Completions>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto __with_cuda_error(_Completions __completions) noexcept
|
||||
{
|
||||
return __completions - __eptr_completion() + completion_signatures<set_error_t(cudaError_t)>{};
|
||||
}
|
||||
|
||||
template <class _Config>
|
||||
using __dims_of_t = typename _Config::hierarchy_type;
|
||||
|
||||
// This kernel forwards the results from the child sender to the receiver of the parent
|
||||
// sender. The receiver is where most algorithms do their work, so we want the receiver to
|
||||
// tell us how to launch the kernel that completes it. Thus, the launch configuration is
|
||||
// read from the outer receiver's environment.
|
||||
template <int _ThreadsPerBlock, class _Rcvr, class _Variant>
|
||||
_CCCL_VISIBILITY_HIDDEN __launch_bounds__(_ThreadsPerBlock) __global__
|
||||
void __completion_kernel(__state_base_t<_Rcvr, _Variant>* __state)
|
||||
{
|
||||
_CCCL_ASSERT(__state->__results_.__index() != __npos, "__completion_kernel called with empty results");
|
||||
__visit(__visit_results{}, __state->__results_, __state->__rcvr_);
|
||||
}
|
||||
|
||||
// This is the environment of the inner receiver that is used to connect the child sender.
|
||||
template <class _Env, class _Config>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __env_t
|
||||
{
|
||||
_CCCL_TEMPLATE(class _Query, class... _As)
|
||||
_CCCL_REQUIRES(__queryable_with<_Env, _Query, _As...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(_Query, _As&&... __args) const
|
||||
noexcept(__nothrow_queryable_with<_Env, _Query, _As...>) -> __query_result_t<_Env, _Query, _As...>
|
||||
{
|
||||
return __env_.query(_Query{}, static_cast<_As&&>(__args)...);
|
||||
}
|
||||
|
||||
// This query is used to tell transform_sender that the child sender has been adapted to
|
||||
// the stream domain.
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(__stream::__adapted_t) const noexcept
|
||||
-> ::cuda::std::__ignore_t
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_launch_config_t) const noexcept -> _Config
|
||||
{
|
||||
return __launch_config_;
|
||||
}
|
||||
|
||||
_Env __env_;
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ _Config __launch_config_;
|
||||
};
|
||||
|
||||
// This is the inner receiver that is used to connect the child sender.
|
||||
template <class _Rcvr, class _Config, class _Variant>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_t
|
||||
{
|
||||
template <class _Tag, class... _Args>
|
||||
_CCCL_HOST_DEVICE_API void __complete(_Tag, _Args&&... __args) noexcept
|
||||
{
|
||||
if (__state_->__complete_inline_) // TODO: untested
|
||||
{
|
||||
_Tag{}(static_cast<_Rcvr&&>(__state_->__rcvr_), static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
else
|
||||
{
|
||||
using __tuple_t = ::cuda::std::__decayed_tuple<_Tag, _Args...>;
|
||||
__state_->__results_.template __emplace<__tuple_t>(_Tag{}, static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
}
|
||||
|
||||
template <class... _Args>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_value(_Args&&... __args) noexcept
|
||||
{
|
||||
__complete(execution::set_value, static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_error(_Error&& __err) noexcept
|
||||
{
|
||||
// Map any exception_ptr error completions to cudaErrorUnknown:
|
||||
if constexpr (__same_as<::cuda::std::remove_cvref_t<_Error>, exception_ptr>)
|
||||
{
|
||||
__complete(execution::set_error, cudaErrorUnknown);
|
||||
}
|
||||
else
|
||||
{
|
||||
__complete(execution::set_error, static_cast<_Error&&>(__err));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_stopped() noexcept
|
||||
{
|
||||
__complete(execution::set_stopped);
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __env_t<env_of_t<_Rcvr>, _Config>
|
||||
{
|
||||
return {execution::get_env(__state_->__rcvr_), __state_->__launch_config_};
|
||||
}
|
||||
|
||||
__state_t<_Rcvr, _Config, _Variant>* __state_;
|
||||
};
|
||||
|
||||
template <class _CvSndr, class _Rcvr, class _GetStream>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __opstate_t(_CvSndr&& __sndr, _Rcvr __rcvr, _GetStream __get_stream)
|
||||
: __stream_{__get_stream(__sndr, execution::get_env(__rcvr))}
|
||||
{
|
||||
NV_IF_ELSE_TARGET(NV_IS_HOST,
|
||||
(__host_make_state(static_cast<_CvSndr&&>(__sndr), static_cast<_Rcvr&&>(__rcvr));),
|
||||
(__device_make_state(static_cast<_CvSndr&&>(__sndr), static_cast<_Rcvr&&>(__rcvr));));
|
||||
}
|
||||
|
||||
_CCCL_IMMOVABLE(__opstate_t);
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void start() noexcept
|
||||
{
|
||||
NV_IF_ELSE_TARGET(NV_IS_HOST, ({ __host_start(); }), ({ __device_start(); }));
|
||||
}
|
||||
|
||||
// This is called by the continues_on adaptor after it has sync'ed the stream.
|
||||
template <class _Rcvr2>
|
||||
_CCCL_HOST_DEVICE_API auto __set_results(_Rcvr2& __rcvr) noexcept
|
||||
{
|
||||
__visit(__visit_results{}, __get_state().__state_.__results_, __rcvr);
|
||||
}
|
||||
|
||||
private:
|
||||
using __sndr_config_t = __call_result_t<get_launch_config_t, env_of_t<_CvSndr>>;
|
||||
using __rcvr_config_t = __call_result_t<get_launch_config_t, env_of_t<_Rcvr>>;
|
||||
using __env_t = __stream::__env_t<env_of_t<_Rcvr>, __sndr_config_t>;
|
||||
using __child_completions_t = completion_signatures_of_t<_CvSndr, __env_t>;
|
||||
using __completions_t = decltype(__stream::__with_cuda_error(__child_completions_t{}));
|
||||
using __results_t = typename __completions_t::template __transform_q<::cuda::std::__decayed_tuple, __variant>;
|
||||
using __rcvr_t = __stream::__rcvr_t<_Rcvr, __sndr_config_t, __results_t>;
|
||||
|
||||
_CCCL_HOST_API void __host_make_state(_CvSndr&& __sndr, _Rcvr __rcvr)
|
||||
{
|
||||
// If *this is already in device or managed memory, then we can avoid a separate
|
||||
// allocation.
|
||||
if (auto const __attrs = execution::__get_pointer_attributes(this); __attrs.type == ::cudaMemoryTypeManaged)
|
||||
{
|
||||
__state_.template __emplace<__state_t>(static_cast<_CvSndr&&>(__sndr), static_cast<_Rcvr&&>(__rcvr));
|
||||
}
|
||||
else
|
||||
{
|
||||
__state_.__emplace(
|
||||
__managed_box<__state_t>::__make_unique(static_cast<_CvSndr&&>(__sndr), static_cast<_Rcvr&&>(__rcvr)));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_DEVICE_API void __device_make_state(_CvSndr&& __sndr, _Rcvr __rcvr)
|
||||
{
|
||||
__state_.template __emplace<__state_t>(static_cast<_CvSndr&&>(__sndr), static_cast<_Rcvr&&>(__rcvr));
|
||||
}
|
||||
|
||||
_CCCL_HOST_API void __host_start() noexcept
|
||||
{
|
||||
auto& __state = __get_state();
|
||||
|
||||
// Read the launch configuration passed to us by the parent operation. When we launch
|
||||
// the completion kernel, we will be completing the parent's receiver, so we must let
|
||||
// the receiver tell us how to launch the kernel.
|
||||
auto const __launch_config = get_launch_config(execution::get_env(__state.__state_.__rcvr_));
|
||||
constexpr auto __threads_per_block = gpu_thread.count_as<int>(block, __launch_config);
|
||||
|
||||
// Start the child operation state. This will launch kernels for all the predecessors
|
||||
// of this operation.
|
||||
execution::start(__state.__opstate_);
|
||||
|
||||
_CCCL_TRY
|
||||
{
|
||||
// launch a kernel to pass the results to the receiver.
|
||||
auto* __kernel = &__completion_kernel<__threads_per_block, _Rcvr, __results_t>;
|
||||
::cuda::launch(__stream_, __launch_config, __kernel, &__state.__state_);
|
||||
}
|
||||
_CCCL_CATCH (::cuda::cuda_error & __error) // Check for errors in the kernel launch.
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(__state.__state_.__rcvr_), __error.status());
|
||||
}
|
||||
_CCCL_CATCH_ALL
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(__state.__state_.__rcvr_), cudaErrorUnknown);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: untested
|
||||
_CCCL_DEVICE_API void __device_start() noexcept
|
||||
{
|
||||
auto& __state = __get_state();
|
||||
|
||||
auto const __launch_config = get_launch_config(execution::get_env(__state.__state_.__rcvr_));
|
||||
constexpr auto __threads_per_block = gpu_thread.count_as<int>(block, __launch_config);
|
||||
|
||||
// without the following, the kernel in __host_start will fail to launch with
|
||||
// cudaErrorInvalidDeviceFunction.
|
||||
#ifndef _CCCL_CLANG_TIDY_INVOKED
|
||||
// clang-tidy<22 errors when compiling this, complaining that we are taking a reference to
|
||||
// __global__ function inside a __device__ function.
|
||||
::__cccl_unused(&__completion_kernel<__threads_per_block, _Rcvr, __results_t>);
|
||||
#endif
|
||||
__state.__state_.__complete_inline_ = true;
|
||||
execution::start(__state.__opstate_);
|
||||
}
|
||||
|
||||
// This is the part of the operation state that is stored in managed memory.
|
||||
struct __state_t
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __state_t(_CvSndr&& __sndr, _Rcvr __rcvr)
|
||||
: __state_{{static_cast<_Rcvr&&>(__rcvr), {}, false}, get_launch_config(execution::get_env(__sndr))}
|
||||
, __opstate_(execution::connect(static_cast<_CvSndr&&>(__sndr), __rcvr_t{&__state_}))
|
||||
{}
|
||||
|
||||
__stream::__state_t<_Rcvr, __sndr_config_t, __results_t> __state_;
|
||||
connect_result_t<_CvSndr, __rcvr_t> __opstate_;
|
||||
};
|
||||
|
||||
// Return a reference to the state for this operation, whether it is stored in-situ or
|
||||
// in dyncamically-allocated managed memory.
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __get_state() noexcept -> __state_t&
|
||||
{
|
||||
return __state_.__index() == 0
|
||||
? execution::__variant_get<0>(__state_)
|
||||
: execution::__variant_get<1>(__state_)->__value;
|
||||
}
|
||||
|
||||
stream_ref __stream_;
|
||||
__variant<__state_t, ::cuda::std::unique_ptr<__managed_box<__state_t>>> __state_{};
|
||||
};
|
||||
|
||||
template <class _Sndr, class _GetStream>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
template <class _Sndr, class _GetStream>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __attrs_t
|
||||
{
|
||||
// If the child sender knows how to provide a stream, make it available via the stream
|
||||
// adapter's attributes.
|
||||
_CCCL_TEMPLATE(class _GetStream2 = _GetStream)
|
||||
_CCCL_REQUIRES(__callable<_GetStream2, _Sndr, env<>>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_stream_t) const noexcept -> stream_ref
|
||||
{
|
||||
return __sndr_.__get_stream_(__sndr_.__sndr_, env{});
|
||||
}
|
||||
|
||||
// This sender executes asynchronously with respect to 'start()':
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_behavior_t) const noexcept
|
||||
{
|
||||
return completion_behavior::asynchronous;
|
||||
}
|
||||
|
||||
// This forwards even non-forwarding queries. A stream sender adaptor is not an ordinary
|
||||
// sender adaptor, like `then` or `let_value`. A stream sender adaptor is an
|
||||
// implementation detail that is not visible to the user. It should be as transparent as
|
||||
// possible.
|
||||
_CCCL_TEMPLATE(class _Query, class... _Args)
|
||||
_CCCL_REQUIRES(__queryable_with<env_of_t<_Sndr>, _Query, _Args...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(_Query, _Args&&... __args) const
|
||||
noexcept(__nothrow_queryable_with<env_of_t<_Sndr>, _Query, _Args...>)
|
||||
-> __query_result_t<env_of_t<_Sndr>, _Query, _Args...>
|
||||
{
|
||||
return execution::get_env(__sndr_.__sndr_).query(_Query{}, static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
const __sndr_t<_Sndr, _GetStream>& __sndr_;
|
||||
};
|
||||
|
||||
// This is the sender adaptor that adapts a non-stream sender to a stream sender.
|
||||
template <class _Sndr, class _GetStream>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class _Self, class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures() noexcept
|
||||
{
|
||||
using __cv_sndr_t _CCCL_NODEBUG_ALIAS = ::cuda::std::__copy_cvref_t<_Self, _Sndr>;
|
||||
using __sndr_config_t _CCCL_NODEBUG_ALIAS = __call_result_t<get_launch_config_t, env_of_t<_Sndr>>;
|
||||
using __env_t = __stream::__env_t<_Env, __sndr_config_t>;
|
||||
_CUDAX_LET_COMPLETIONS(auto(__completions) = execution::get_completion_signatures<__cv_sndr_t, __env_t>())
|
||||
{
|
||||
return __stream::__with_cuda_error(__completions);
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) && -> __opstate_t<_Sndr, _Rcvr, _GetStream>
|
||||
{
|
||||
return __opstate_t<_Sndr, _Rcvr, _GetStream>(
|
||||
static_cast<_Sndr&&>(__sndr_), static_cast<_Rcvr&&>(__rcvr), __get_stream_);
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
connect(_Rcvr __rcvr) const& -> __opstate_t<const _Sndr&, _Rcvr, _GetStream>
|
||||
{
|
||||
return __opstate_t<const _Sndr&, _Rcvr, _GetStream>(__sndr_, static_cast<_Rcvr&&>(__rcvr), __get_stream_);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __attrs_t<_Sndr, _GetStream>
|
||||
{
|
||||
return __attrs_t<_Sndr, _GetStream>{*this};
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ __tag_t<__stream::__tag_of_t<_Sndr>> __tag_;
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ _GetStream __get_stream_;
|
||||
_Sndr __sndr_;
|
||||
};
|
||||
|
||||
template <class _Sndr, class _GetStream>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto
|
||||
__adapt(_Sndr&& __sndr, _GetStream __get_stream) noexcept(__nothrow_decay_copyable<_Sndr>)
|
||||
{
|
||||
return __sndr_t<_Sndr, _GetStream>{{}, __get_stream, static_cast<_Sndr&&>(__sndr)};
|
||||
}
|
||||
} // namespace __stream
|
||||
|
||||
template <class _Sndr, class _GetStream>
|
||||
inline constexpr int structured_binding_size<__stream::__sndr_t<_Sndr, _GetStream>> = 3;
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_STREAM_ADAPTOR
|
||||
@@ -1,226 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_STREAM_BULK
|
||||
#define __CUDAX_EXECUTION_STREAM_BULK
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__type_traits/is_specialization_of.h>
|
||||
#include <cuda/std/__tuple_dir/ignore.h>
|
||||
#include <cuda/std/__utility/forward_like.h>
|
||||
|
||||
#include <cuda/experimental/__execution/bulk.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/policy.cuh>
|
||||
#include <cuda/experimental/__execution/stream/domain.cuh>
|
||||
#include <cuda/experimental/launch.cuh>
|
||||
|
||||
#include <cooperative_groups.h>
|
||||
#include <cuda_runtime_api.h>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
namespace __stream
|
||||
{
|
||||
struct __bulk_chunked_t : execution::__bulk_t<__bulk_chunked_t>
|
||||
{
|
||||
template <class _Shape, class _Fn, class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_t : __bulk_t::__rcvr_base_t<_Shape, _Fn, _Rcvr>
|
||||
{
|
||||
// We permit this `set_value` function to be called multiple times, once for each
|
||||
// thread in the block.
|
||||
template <class... _Values>
|
||||
_CCCL_DEVICE_API void set_value(_Values&&... __values) noexcept
|
||||
{
|
||||
const _Shape __tid = threadIdx.x + blockIdx.x * blockDim.x;
|
||||
|
||||
if (__tid < this->__state_->__shape_)
|
||||
{
|
||||
if constexpr (::cuda::__is_specialization_of_v<_Fn, bulk_t::__bulk_chunked_fn>)
|
||||
{
|
||||
// If the chunked function was adapted from an unchunked function, we can call
|
||||
// the unchunked functions directly.
|
||||
this->__state_->__fn_.__fn_(_Shape(__tid), __values...);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Otherwise, we call the function with the half-open range [__tid, __tid + 1)
|
||||
// to process a single element.
|
||||
this->__state_->__fn_(_Shape(__tid), _Shape(__tid + 1), __values...);
|
||||
}
|
||||
}
|
||||
|
||||
::cooperative_groups::this_grid().sync();
|
||||
|
||||
// Only call the downstream receiver once, after all threads have processed their
|
||||
// elements.
|
||||
if (__tid == 0)
|
||||
{
|
||||
execution::set_value(static_cast<_Rcvr&&>(this->__state_->__rcvr_), static_cast<_Values&&>(__values)...);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Sndr, class _Policy, class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t : __bulk_t::__sndr_base_t<_Sndr, _Policy, _Shape, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Policy, class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t : __bulk_t::__closure_base_t<_Policy, _Shape, _Fn>
|
||||
{};
|
||||
|
||||
// This function is called when the `bulk_chunked` CPO calls `transform_sender` with a
|
||||
// domain argument of stream_domain. It adapts a `bulk_chunked` sender to the stream
|
||||
// domain.
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(set_value_t, _Sndr&& __sndr, ::cuda::std::__ignore_t) const
|
||||
{
|
||||
// Decompose the bulk sender into its components:
|
||||
auto& [__tag, __state, __child] = __sndr;
|
||||
auto& [__policy, __shape, __fn] = __state;
|
||||
|
||||
using __policy_t = decltype(__policy);
|
||||
using __shape_t = decltype(__shape);
|
||||
using __fn_t = decltype(__fn);
|
||||
using __sndr_t = __bulk_chunked_t::__sndr_t<decltype(__child), __policy_t, __shape_t, __fn_t>;
|
||||
using __closure_t = __bulk_t::__closure_base_t<__policy_t, __shape_t, __fn_t>;
|
||||
|
||||
auto __closure = __closure_t{__policy, __shape, ::cuda::std::forward_like<_Sndr>(__fn)};
|
||||
auto __new_sndr = __sndr_t{{{}, static_cast<__closure_t&&>(__closure), ::cuda::std::forward_like<_Sndr>(__child)}};
|
||||
return __stream::__adapt(static_cast<__sndr_t&&>(__new_sndr));
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API static constexpr bool __is_chunked() noexcept
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __bulk_unchunked_t : execution::__bulk_t<__bulk_unchunked_t>
|
||||
{
|
||||
template <class _Shape, class _Fn, class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_t : __bulk_t::__rcvr_base_t<_Shape, _Fn, _Rcvr>
|
||||
{
|
||||
// We permit this `set_value` function to be called multiple times, once for each
|
||||
// thread in the block.
|
||||
template <class... _Values>
|
||||
_CCCL_DEVICE_API void set_value(_Values&&... __values) noexcept
|
||||
{
|
||||
const _Shape __tid = threadIdx.x + blockIdx.x * blockDim.x;
|
||||
|
||||
// Each thread processes exactly one element, if it is in range.
|
||||
if (__tid < this->__state_->__shape_)
|
||||
{
|
||||
this->__state_->__fn_(_Shape(__tid), __values...);
|
||||
}
|
||||
|
||||
::cooperative_groups::this_grid().sync();
|
||||
|
||||
// Only call the downstream receiver once, after all threads have processed their
|
||||
// elements.
|
||||
if (__tid == 0)
|
||||
{
|
||||
execution::set_value(static_cast<_Rcvr&&>(this->__state_->__rcvr_), static_cast<_Values&&>(__values)...);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Sndr, class _Policy, class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t : __bulk_t::__sndr_base_t<_Sndr, _Policy, _Shape, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Policy, class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t : __bulk_t::__closure_base_t<_Policy, _Shape, _Fn>
|
||||
{};
|
||||
|
||||
// This function is called when the `bulk_unchunked` CPO calls `transform_sender` with a
|
||||
// domain argument of stream_domain. It adapts a `bulk_unchunked` sender to the stream
|
||||
// domain.
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(set_value_t, _Sndr&& __sndr, ::cuda::std::__ignore_t) const
|
||||
{
|
||||
// Decompose the bulk sender into its components:
|
||||
auto& [__tag, __state, __child] = __sndr;
|
||||
auto& [__policy, __shape, __fn] = __state;
|
||||
|
||||
using __policy_t = decltype(__policy);
|
||||
using __shape_t = decltype(__shape);
|
||||
using __fn_t = decltype(__fn);
|
||||
using __sndr_t = __bulk_unchunked_t::__sndr_t<decltype(__child), __policy_t, __shape_t, __fn_t>;
|
||||
using __closure_t = __bulk_t::__closure_base_t<__policy_t, __shape_t, __fn_t>;
|
||||
|
||||
auto __closure = __closure_t{__policy, __shape, ::cuda::std::forward_like<_Sndr>(__fn)};
|
||||
auto __new_sndr = __sndr_t{{{}, static_cast<__closure_t&&>(__closure), ::cuda::std::forward_like<_Sndr>(__child)}};
|
||||
return __stream::__adapt(static_cast<__sndr_t&&>(__new_sndr));
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API static constexpr bool __is_chunked() noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
struct __bulk_t : execution::__bulk_t<__bulk_t>
|
||||
{
|
||||
template <class _Sndr, class _Policy, class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t : __bulk_t::__sndr_base_t<_Sndr, _Policy, _Shape, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Policy, class _Shape, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t : __bulk_t::__closure_base_t<_Policy, _Shape, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(set_value_t, _Sndr&& __sndr, ::cuda::std::__ignore_t) const
|
||||
-> decltype(auto)
|
||||
{
|
||||
// This converts a bulk sender into a bulk_chunked sender, which will then be
|
||||
// further transformed by __bulk_chunked_t above.
|
||||
return bulk.transform_sender(set_value, static_cast<_Sndr&&>(__sndr), env{});
|
||||
}
|
||||
};
|
||||
} // namespace __stream
|
||||
|
||||
template <>
|
||||
struct stream_domain::__apply_t<bulk_chunked_t> : __stream::__bulk_chunked_t
|
||||
{};
|
||||
|
||||
template <>
|
||||
struct stream_domain::__apply_t<bulk_unchunked_t> : __stream::__bulk_unchunked_t
|
||||
{};
|
||||
|
||||
template <>
|
||||
struct stream_domain::__apply_t<bulk_t> : __stream::__bulk_t
|
||||
{};
|
||||
|
||||
template <class _Sndr, class _Policy, class _Shape, class _Fn>
|
||||
inline constexpr int structured_binding_size<__stream::__bulk_chunked_t::__sndr_t<_Sndr, _Policy, _Shape, _Fn>> = 3;
|
||||
|
||||
template <class _Sndr, class _Policy, class _Shape, class _Fn>
|
||||
inline constexpr int structured_binding_size<__stream::__bulk_unchunked_t::__sndr_t<_Sndr, _Policy, _Shape, _Fn>> = 3;
|
||||
|
||||
template <class _Sndr, class _Policy, class _Shape, class _Fn>
|
||||
inline constexpr int structured_binding_size<__stream::__bulk_t::__sndr_t<_Sndr, _Policy, _Shape, _Fn>> = 3;
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_STREAM_BULK
|
||||
@@ -1,65 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_STREAM_CONTEXT_IMPL
|
||||
#define __CUDAX_EXECUTION_STREAM_CONTEXT_IMPL
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__device/device_ref.h>
|
||||
#include <cuda/__stream/get_stream.h>
|
||||
#include <cuda/__utility/immovable.h>
|
||||
|
||||
#include <cuda/experimental/__execution/stream/scheduler.cuh>
|
||||
#include <cuda/experimental/stream.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// stream_context
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT stream_context : private __immovable
|
||||
{
|
||||
_CCCL_HOST_API explicit stream_context(device_ref __device)
|
||||
: __stream_{__device}
|
||||
{}
|
||||
|
||||
_CCCL_HOST_API void sync() noexcept
|
||||
{
|
||||
__stream_.sync();
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_stream_t) const noexcept -> stream_ref
|
||||
{
|
||||
return __stream_;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto get_scheduler() noexcept -> stream_scheduler
|
||||
{
|
||||
return stream_scheduler{__stream_};
|
||||
}
|
||||
|
||||
private:
|
||||
stream __stream_;
|
||||
};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_STREAM_CONTEXT_IMPL
|
||||
@@ -1,186 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_STREAM_DOMAIN
|
||||
#define __CUDAX_EXECUTION_STREAM_DOMAIN
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__stream/get_stream.h>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__functional/compose.h>
|
||||
#include <cuda/std/__type_traits/is_callable.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/domain.cuh>
|
||||
#include <cuda/experimental/__execution/queries.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__stream/stream_ref.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
namespace __stream
|
||||
{
|
||||
template <class _Tag>
|
||||
struct __tag_t
|
||||
{};
|
||||
|
||||
struct __no_tag_t
|
||||
{};
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto __tag_of(::cuda::std::__ignore_t) -> __no_tag_t;
|
||||
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API auto __tag_of(const _Sndr& __sndr) -> tag_of_t<_Sndr>;
|
||||
|
||||
template <class _Sndr>
|
||||
using __tag_of_t = decltype(__stream::__tag_of(declval<_Sndr>()));
|
||||
|
||||
struct __adapted_t
|
||||
{};
|
||||
|
||||
template <class _Sndr, class _GetStream>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto __get_stream_from_attrs =
|
||||
__first_callable{get_stream, ::cuda::std::__compose(get_stream, get_completion_scheduler<set_value_t>)};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto __get_stream_from_env =
|
||||
__first_callable{get_stream, ::cuda::std::__compose(get_stream, get_scheduler)};
|
||||
|
||||
using __get_stream_from_attrs_t = decltype(__get_stream_from_attrs);
|
||||
using __get_stream_from_env_t = decltype(__get_stream_from_env);
|
||||
|
||||
// Get the stream either the sender's attributes or from the receiver's environment.
|
||||
struct __get_stream_fn
|
||||
{
|
||||
_CCCL_TEMPLATE(class _Sndr, class _Env)
|
||||
_CCCL_REQUIRES((__callable<__get_stream_from_attrs_t, env_of_t<_Sndr>, const _Env&>
|
||||
|| __callable<__get_stream_from_env_t, _Env>) )
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(const _Sndr& __sndr, const _Env& __env) const noexcept -> stream_ref
|
||||
{
|
||||
if constexpr (__callable<__get_stream_from_attrs_t, env_of_t<_Sndr>, const _Env&>)
|
||||
{
|
||||
// If the sender's attributes have a stream, use it.
|
||||
return __get_stream_from_attrs(execution::get_env(__sndr), __env);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Otherwise, try to get the stream from the receiver's environment.
|
||||
return __get_stream_from_env(__env);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Forward declaration of the __adapt function
|
||||
template <class _Sndr, class _GetStream = __get_stream_fn>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto __adapt(_Sndr&& __sndr, _GetStream = {}) noexcept(__nothrow_decay_copyable<_Sndr>);
|
||||
} // namespace __stream
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto __get_stream = __stream::__get_stream_fn{};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// stream domain
|
||||
struct stream_domain
|
||||
{
|
||||
_CUDAX_SEMI_PRIVATE :
|
||||
struct __apply_adapt_t
|
||||
{
|
||||
// This is the default apply function that adapts a sender to a stream sender.
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto
|
||||
operator()(::cuda::std::__ignore_t, _Sndr&& __sndr, ::cuda::std::__ignore_t) const
|
||||
noexcept(__nothrow_decay_copyable<_Sndr>)
|
||||
{
|
||||
return __stream::__adapt(static_cast<_Sndr&&>(__sndr));
|
||||
}
|
||||
};
|
||||
|
||||
struct __apply_passthru_t
|
||||
{
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto
|
||||
operator()(::cuda::std::__ignore_t, _Sndr&& __sndr, ::cuda::std::__ignore_t) const
|
||||
noexcept(__nothrow_movable<_Sndr>) -> _Sndr
|
||||
{
|
||||
return static_cast<_Sndr&&>(__sndr);
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Tag>
|
||||
struct __apply_t : __apply_adapt_t
|
||||
{};
|
||||
|
||||
template <class _Sndr, class _Env>
|
||||
_CCCL_HOST_DEVICE_API static constexpr auto __transform_strategy() noexcept
|
||||
{
|
||||
if constexpr (__queryable_with<_Env, __stream::__adapted_t>)
|
||||
{
|
||||
// The __stream::__adapted_t query is present only on receivers that come from an
|
||||
// adapted sender. Therefore, _Sndr has already been adapted. Pass it through as is.
|
||||
return __apply_passthru_t{};
|
||||
}
|
||||
else if constexpr (sender_for<_Sndr>)
|
||||
{
|
||||
// The sender has a tag type. Use the tag to determine the transformation to apply.
|
||||
return __apply_t<tag_of_t<_Sndr>>{};
|
||||
}
|
||||
else
|
||||
{
|
||||
// Otherwise, _Sndr is an unknown sender type that has not yet been adapted to
|
||||
// be a stream sender. Adapt it now.
|
||||
return __apply_adapt_t{};
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Sndr, class _Env>
|
||||
using __transform_strategy_t = decltype(__transform_strategy<_Sndr, _Env>());
|
||||
|
||||
public:
|
||||
_CCCL_TEMPLATE(class _Tag, class _Sndr, class... _Args)
|
||||
_CCCL_REQUIRES(__callable<__apply_t<_Tag>, _Sndr, _Args...>)
|
||||
_CCCL_HOST_DEVICE_API static constexpr auto
|
||||
apply_sender(_Tag, _Sndr&& __sndr, _Args&&... __args) noexcept(__nothrow_callable<__apply_t<_Tag>, _Sndr, _Args...>)
|
||||
-> __call_result_t<__apply_t<_Tag>, _Sndr, _Args...>
|
||||
{
|
||||
return __apply_t<_Tag>()(static_cast<_Sndr&&>(__sndr), static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _OpTag, class _Sndr, class _Env, class _Apply = __transform_strategy_t<_Sndr, _Env>)
|
||||
_CCCL_REQUIRES(__callable<_Apply, _OpTag, _Sndr, const _Env&>)
|
||||
_CCCL_HOST_DEVICE_API static constexpr auto transform_sender(_OpTag, _Sndr&& __sndr, const _Env& __env) noexcept(
|
||||
__nothrow_callable<_Apply, _OpTag, _Sndr, const _Env&>) -> __call_result_t<_Apply, _OpTag, _Sndr, const _Env&>
|
||||
{
|
||||
return _Apply()(_OpTag(), static_cast<_Sndr&&>(__sndr), __env);
|
||||
}
|
||||
};
|
||||
|
||||
// If a sender has already been adapted to a stream sender, it will have a tag that is a specialization of
|
||||
// __stream::__tag_t. In that case, we don't need to adapt it again, and we can just pass it through.
|
||||
template <class _Tag>
|
||||
struct stream_domain::__apply_t<__stream::__tag_t<_Tag>> : stream_domain::__apply_passthru_t
|
||||
{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_STREAM_DOMAIN
|
||||
@@ -1,46 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_STREAM_LAUNCH
|
||||
#define __CUDAX_EXECUTION_STREAM_LAUNCH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/experimental/__execution/stream/domain.cuh>
|
||||
#include <cuda/experimental/__launch/launch.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
// TODO: not implemented yet
|
||||
template <>
|
||||
struct stream_domain::__apply_t<__kernel_t>
|
||||
{
|
||||
template <class _Sndr, class _Env>
|
||||
_CCCL_HOST_DEVICE_API auto operator()(_Sndr __sndr, const _Env& __env) const
|
||||
{
|
||||
static_assert(::cuda::std::__always_false_v<_Sndr>,
|
||||
"The CUDA stream scheduler does not yet support the `launch` algorithm.");
|
||||
}
|
||||
};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_STREAM_LAUNCH
|
||||
@@ -1,73 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_STREAM_LET_VALUE
|
||||
#define __CUDAX_EXECUTION_STREAM_LET_VALUE
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/let_value.cuh>
|
||||
#include <cuda/experimental/__execution/stream/domain.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
// let_value, let_error, let_stopped: customization for the stream scheduler
|
||||
template <>
|
||||
struct stream_domain::__apply_t<let_value_t>
|
||||
{
|
||||
template <class _Sndr, class _Env>
|
||||
_CCCL_HOST_DEVICE_API auto operator()(_Sndr __sndr, const _Env& __env) const
|
||||
{
|
||||
static_assert(::cuda::std::__always_false_v<_Sndr>,
|
||||
"The CUDA stream scheduler does not yet support the `let_value`, `let_error`, and `let_stopped` "
|
||||
"algorithms.");
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct stream_domain::__apply_t<let_error_t>
|
||||
{
|
||||
template <class _Sndr, class _Env>
|
||||
_CCCL_HOST_DEVICE_API auto operator()(_Sndr __sndr, const _Env& __env) const
|
||||
{
|
||||
static_assert(::cuda::std::__always_false_v<_Sndr>,
|
||||
"The CUDA stream scheduler does not yet support the `let_value`, `let_error`, and `let_stopped` "
|
||||
"algorithms.");
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct stream_domain::__apply_t<let_stopped_t>
|
||||
{
|
||||
template <class _Sndr, class _Env>
|
||||
_CCCL_HOST_DEVICE_API auto operator()(_Sndr __sndr, const _Env& __env) const
|
||||
{
|
||||
static_assert(::cuda::std::__always_false_v<_Sndr>,
|
||||
"The CUDA stream scheduler does not yet support the `let_value`, `let_error`, and `let_stopped` "
|
||||
"algorithms.");
|
||||
}
|
||||
};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_STREAM_LET_VALUE
|
||||
@@ -1,231 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_STREAM_CONTINUES_ON
|
||||
#define __CUDAX_EXECUTION_STREAM_CONTINUES_ON
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__stream/get_stream.h>
|
||||
#include <cuda/__type_traits/is_specialization_of.h>
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__utility/forward_like.h>
|
||||
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/rcvr_ref.cuh>
|
||||
#include <cuda/experimental/__execution/schedule_from.cuh>
|
||||
#include <cuda/experimental/__execution/stream/adaptor.cuh>
|
||||
#include <cuda/experimental/__execution/stream/domain.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
#include <cuda/experimental/__launch/launch.cuh>
|
||||
#include <cuda/experimental/__stream/stream_ref.cuh>
|
||||
|
||||
#include <cuda_runtime_api.h>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
namespace __stream
|
||||
{
|
||||
//! The customization of schedule_from, when transferring back to the CPU, involves
|
||||
//! adapting the sender and receiver types.
|
||||
//!
|
||||
//! A schedule_from sender such as schedule_from(sndr), where sndr completes on the GPU,
|
||||
//! needs to synchronize the CUDA stream to ensure that all queued GPU work is finished.
|
||||
//! Only then can the schedule operation be safely invoked -- from the CPU.
|
||||
//!
|
||||
//! To effect this, schedule_from(sndr) is transformed into
|
||||
//! schedule_from(SYNC-STREAM-ADAPTOR(sndr)), where SYNC-STREAM-ADAPTOR(sndr) is a
|
||||
//! sender that does the following:
|
||||
//!
|
||||
//! 1. In connect (called on host): Connects sndr with a sink receiver that ignores values
|
||||
//! passed to it and simply returns. The sink receiver's completion operations are
|
||||
//! executed on device when the child sender completes.
|
||||
//!
|
||||
//! 2. In start (called on host): Starts the child sender, which launches kernels for the
|
||||
//! predecessor operations, and then synchronizes the CUDA stream to ensure all queued
|
||||
//! GPU work is finished. Then, it pulls the results from sndr's operation state and
|
||||
//! passes them to the receiver on the host.
|
||||
struct __schedule_from_t
|
||||
{
|
||||
// Transition from the GPU to the CPU domain
|
||||
template <class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_t
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
template <class... _Values>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_value(_Values&&...) noexcept
|
||||
{
|
||||
// no-op
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_error(::cuda::std::__ignore_t) noexcept
|
||||
{
|
||||
// no-op
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_stopped() noexcept
|
||||
{
|
||||
// no-op
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __fwd_env_t<env_of_t<_Rcvr>>
|
||||
{
|
||||
return __fwd_env(execution::get_env(__rcvr_));
|
||||
}
|
||||
|
||||
_Rcvr& __rcvr_;
|
||||
};
|
||||
|
||||
// This opstate will be stored in host memory.
|
||||
template <class _Sndr, class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
using __env_t = __fwd_env_t<env_of_t<_Rcvr>>;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __opstate_t(_Sndr&& __sndr, _Rcvr __rcvr)
|
||||
: __rcvr_(static_cast<_Rcvr&&>(__rcvr))
|
||||
, __stream_(__get_stream(__sndr, execution::get_env(__rcvr_)))
|
||||
, __opstate_(execution::connect(static_cast<_Sndr&&>(__sndr), __rcvr_t<_Rcvr>{__rcvr_}))
|
||||
{}
|
||||
|
||||
_CCCL_IMMOVABLE(__opstate_t);
|
||||
|
||||
_CCCL_HOST_DEVICE_API void start() noexcept
|
||||
{
|
||||
NV_IF_ELSE_TARGET(NV_IS_HOST, ({ __host_start(); }), ({ __device_start(); }));
|
||||
}
|
||||
|
||||
_CCCL_HOST_API void __host_start() noexcept
|
||||
{
|
||||
// This launches all predecessor kernels on the given stream
|
||||
execution::start(__opstate_);
|
||||
|
||||
// Synchronize the CUDA stream to make sure all predecessor work has completed, and
|
||||
// the results are available in __opstate_.
|
||||
if (auto __status = ::cudaStreamSynchronize(__stream_.get()); __status != ::cudaSuccess)
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(__rcvr_), cudaError_t(__status));
|
||||
}
|
||||
else
|
||||
{
|
||||
// __opstate_ is an instance of __stream::__opstate_t, and it has a __set_results
|
||||
// member function that will pass the results to the receiver on the host. __rcvr_
|
||||
// is the receiver of the parent default schedule_from operation. That receiver
|
||||
// will then start the schedule operation on the host.
|
||||
__opstate_.__set_results(__rcvr_);
|
||||
}
|
||||
}
|
||||
|
||||
[[noreturn]] _CCCL_DEVICE_API void __device_start() noexcept
|
||||
{
|
||||
_CCCL_ASSERT(false, "internal error: stream::schedule_from opstate started on device");
|
||||
::cuda::std::terminate();
|
||||
|
||||
// We do not want the following to be called, but we need these code paths to be
|
||||
// instantiated. Without this, the __device_start function in stream/adaptor.cuh
|
||||
// will not be instantiated, and the kernel launch in the adaptor's __host_start
|
||||
// function will fail.
|
||||
execution::start(__opstate_);
|
||||
__opstate_.__set_results(__rcvr_);
|
||||
}
|
||||
|
||||
_Rcvr __rcvr_;
|
||||
stream_ref __stream_;
|
||||
connect_result_t<_Sndr, __rcvr_t<_Rcvr>> __opstate_;
|
||||
};
|
||||
|
||||
template <class _Sndr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class _Self, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
return execution::get_child_completion_signatures<_Self, _Sndr, _Env...>();
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) && -> __opstate_t<_Sndr, _Rcvr>
|
||||
{
|
||||
return __opstate_t<_Sndr, _Rcvr>{static_cast<_Sndr&&>(__sndr_), static_cast<_Rcvr&&>(__rcvr)};
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) const& -> __opstate_t<const _Sndr&, _Rcvr>
|
||||
{
|
||||
return __opstate_t<const _Sndr&, _Rcvr>{__sndr_, static_cast<_Rcvr&&>(__rcvr)};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> env_of_t<_Sndr>
|
||||
{
|
||||
return execution::get_env(__sndr_);
|
||||
}
|
||||
|
||||
// The use of __tag_t here instructs the stream_domain not to apply any further
|
||||
// transformations to this sender. See stream/domain.cuh.
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ __tag_t<schedule_from_t> __tag_;
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ ::cuda::std::__ignore_t __ignore_;
|
||||
_Sndr __sndr_;
|
||||
};
|
||||
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API static constexpr auto __mk_sndr(_Sndr&& __sndr)
|
||||
{
|
||||
return __sndr_t<_Sndr>{{}, {}, static_cast<_Sndr&&>(__sndr)};
|
||||
}
|
||||
|
||||
// This function is called when a schedule_from sender, with a predecessor that completes
|
||||
// on the stream scheduler, is being connected. It wraps the child sender so that it
|
||||
// synchronizes the stream after launching the child.
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto operator()(set_value_t, _Sndr&& __sndr, ::cuda::std::__ignore_t) const
|
||||
{
|
||||
static_assert(sender_for<_Sndr, schedule_from_t>);
|
||||
[[maybe_unused]] auto& [__tag, __ign, __child] = __sndr;
|
||||
using __child_t = ::cuda::std::__copy_cvref_t<_Sndr, decltype(__child)>;
|
||||
|
||||
if constexpr (::cuda::__is_specialization_of_v<decltype(__child), __sndr_t>)
|
||||
{
|
||||
return static_cast<_Sndr&&>(__sndr);
|
||||
}
|
||||
else
|
||||
{
|
||||
return execution::schedule_from(__mk_sndr(static_cast<__child_t&&>(__child)));
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace __stream
|
||||
|
||||
template <>
|
||||
struct stream_domain::__apply_t<schedule_from_t> : __stream::__schedule_from_t
|
||||
{};
|
||||
|
||||
template <class _Sndr>
|
||||
inline constexpr int structured_binding_size<__stream::__schedule_from_t::__sndr_t<_Sndr>> = 3;
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_STREAM_CONTINUES_ON
|
||||
@@ -1,300 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_STREAM_SCHEDULER
|
||||
#define __CUDAX_EXECUTION_STREAM_SCHEDULER
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__stream/get_stream.h>
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/queries.cuh>
|
||||
#include <cuda/experimental/__execution/stream/domain.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__stream/stream_ref.cuh>
|
||||
|
||||
#include <cuda_runtime_api.h>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
namespace execution
|
||||
{
|
||||
template <int _ThreadsPerBlock, class _Tag, class _Rcvr>
|
||||
//_CCCL_VISIBILITY_HIDDEN
|
||||
__launch_bounds__(_ThreadsPerBlock) __global__ void __stream_complete(_Tag, _Rcvr* __rcvr)
|
||||
{
|
||||
_Tag{}(static_cast<_Rcvr&&>(*__rcvr));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////
|
||||
// stream scheduler
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT stream_scheduler
|
||||
{
|
||||
using scheduler_concept = scheduler_t;
|
||||
|
||||
_CUDAX_SEMI_PRIVATE:
|
||||
////////////////////////////////////////////////////////////////////////////////////////
|
||||
// attributes of the stream scheduler's sender
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __attrs_t
|
||||
{
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_stream_t) const noexcept -> stream_ref
|
||||
{
|
||||
return __stream_;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_scheduler_t<set_value_t>) const noexcept
|
||||
-> stream_scheduler
|
||||
{
|
||||
return stream_scheduler{__stream_};
|
||||
}
|
||||
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_scheduler_t<set_error_t>, _Env&& __env) const noexcept -> __scheduler_of_t<_Env&>
|
||||
{
|
||||
return execution::get_scheduler(__env);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_TRIVIAL_API constexpr auto query(get_completion_domain_t<set_value_t>) const noexcept
|
||||
-> stream_domain
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_domain_t<set_error_t>, _Env&& __env) const noexcept -> __call_result_t<get_domain_t, _Env&>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_TRIVIAL_API constexpr auto query(get_completion_behavior_t) const noexcept
|
||||
{
|
||||
return completion_behavior::asynchronous;
|
||||
}
|
||||
|
||||
stream_ref __stream_;
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////
|
||||
// stream scheduler's operation state
|
||||
template <class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_HOST_DEVICE_API explicit __opstate_t(_Rcvr __rcvr, stream_ref __stream_ref) noexcept
|
||||
: __rcvr_{static_cast<_Rcvr&&>(__rcvr)}
|
||||
, __stream_{__stream_ref}
|
||||
{
|
||||
NV_IF_TARGET(NV_IS_HOST,
|
||||
(_CCCL_ASSERT(execution::__get_pointer_attributes(this).type == cudaMemoryTypeManaged,
|
||||
"stream scheduler's operation state must be allocated in managed memory");))
|
||||
}
|
||||
|
||||
_CCCL_IMMOVABLE(__opstate_t);
|
||||
|
||||
_CCCL_HOST_DEVICE_API void start() noexcept
|
||||
{
|
||||
NV_IF_ELSE_TARGET(NV_IS_HOST, (__host_start();), (__device_start();));
|
||||
}
|
||||
|
||||
private:
|
||||
_CCCL_HOST_API void __host_start() noexcept
|
||||
{
|
||||
// Read the launch configuration passed to us by the parent operation. When we launch
|
||||
// the completion kernel, we will be completing the parent's receiver, so we must let
|
||||
// the receiver tell us how to launch the kernel.
|
||||
auto const __config = get_launch_config(execution::get_env(__rcvr_));
|
||||
constexpr auto __threads_per_block = cuda::gpu_thread.static_count(cuda::block, __config);
|
||||
const auto __grid_blocks = cuda::block.count_as<unsigned>(cuda::grid, __config);
|
||||
static_assert(__threads_per_block != ::cuda::std::dynamic_extent);
|
||||
|
||||
// Launch the kernel that completes the receiver with the launch configuration from
|
||||
// the receiver.
|
||||
__stream_complete<static_cast<int>(__threads_per_block), set_value_t, _Rcvr>
|
||||
<<<__grid_blocks, static_cast<unsigned>(__threads_per_block), 0, __stream_.get()>>>(set_value, &__rcvr_);
|
||||
|
||||
if (auto __status = cudaGetLastError(); __status != cudaSuccess)
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(__rcvr_), cudaError_t(__status));
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: untested
|
||||
_CCCL_DEVICE_API void __device_start() noexcept
|
||||
{
|
||||
auto __config = get_launch_config(execution::get_env(__rcvr_));
|
||||
constexpr auto __threads_per_block = cuda::gpu_thread.count_as<int>(cuda::block, __config);
|
||||
|
||||
// without the following, the kernel in __host_start will fail to launch with
|
||||
// cudaErrorInvalidDeviceFunction.
|
||||
::cuda::std::ignore = &__stream_complete<__threads_per_block, set_value_t, _Rcvr>;
|
||||
|
||||
execution::set_value(static_cast<_Rcvr&&>(__rcvr_));
|
||||
}
|
||||
|
||||
_Rcvr __rcvr_;
|
||||
stream_ref __stream_;
|
||||
};
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __tag_t
|
||||
{};
|
||||
|
||||
public:
|
||||
_CCCL_HOST_DEVICE_API constexpr stream_scheduler(stream_ref __stream) noexcept
|
||||
: __stream_{__stream}
|
||||
{}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////
|
||||
// stream scheduler's sender
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __sndr_t(stream_ref __stream) noexcept
|
||||
: __attrs_{__stream}
|
||||
{}
|
||||
|
||||
template <class _Self>
|
||||
_CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures() noexcept
|
||||
{
|
||||
return completion_signatures<set_value_t(), set_error_t(cudaError_t)>{};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> const __attrs_t&
|
||||
{
|
||||
return __attrs_;
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto connect(_Rcvr __rcvr) const noexcept -> __opstate_t<_Rcvr>
|
||||
{
|
||||
return __opstate_t<_Rcvr>{static_cast<_Rcvr&&>(__rcvr), __attrs_.__stream_};
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ __tag_t __tag_;
|
||||
__attrs_t __attrs_;
|
||||
};
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_stream_t) const noexcept -> stream_ref
|
||||
{
|
||||
return __stream_;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_domain_t<set_value_t>) const noexcept
|
||||
-> stream_domain
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_domain_t<set_error_t>, _Env&&) const noexcept
|
||||
-> __call_result_t<get_domain_t, _Env&>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_forward_progress_guarantee_t) const noexcept
|
||||
-> forward_progress_guarantee
|
||||
{
|
||||
return forward_progress_guarantee::weakly_parallel;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto schedule() const noexcept -> __sndr_t
|
||||
{
|
||||
return __sndr_t{__stream_};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend bool
|
||||
operator==(const stream_scheduler& __lhs, const stream_scheduler& __rhs) noexcept
|
||||
{
|
||||
return __lhs.__stream_ == __rhs.__stream_;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend bool
|
||||
operator!=(const stream_scheduler& __lhs, const stream_scheduler& __rhs) noexcept
|
||||
{
|
||||
return __lhs.__stream_ != __rhs.__stream_;
|
||||
}
|
||||
|
||||
private:
|
||||
stream_ref __stream_;
|
||||
};
|
||||
|
||||
// The stream_scheduler's sender does not need to be wrapped in a __stream::__sndr_t
|
||||
// because it is already a stream sender. The following specialization ensures that
|
||||
// no transform is applied to the stream_scheduler's sender.
|
||||
template <>
|
||||
struct stream_domain::__apply_t<stream_scheduler::__tag_t> : stream_domain::__apply_passthru_t
|
||||
{};
|
||||
} // namespace execution
|
||||
|
||||
_CCCL_HOST_DEVICE_API inline auto stream_ref::schedule() const noexcept
|
||||
{
|
||||
return execution::schedule(execution::stream_scheduler{*this});
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
stream_ref::query(const execution::get_completion_scheduler_t<execution::set_value_t>&) const noexcept -> stream_ref
|
||||
{
|
||||
return *this;
|
||||
}
|
||||
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto stream_ref::query(
|
||||
const execution::get_completion_scheduler_t<execution::set_error_t>&, const _Env& __env) const noexcept
|
||||
-> execution::__scheduler_of_t<const _Env&>
|
||||
{
|
||||
return execution::get_scheduler(__env);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
stream_ref::query(const execution::get_completion_domain_t<execution::set_value_t>&) const noexcept
|
||||
-> execution::stream_domain
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
stream_ref::query(const execution::get_completion_domain_t<execution::set_error_t>&, const _Env& __env) const noexcept
|
||||
-> __call_result_t<execution::get_domain_t, const _Env&>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
} // namespace cuda::experimental
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_STREAM_SCHEDULER
|
||||
@@ -1,48 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_STREAM_SEQUENCE
|
||||
#define __CUDAX_EXECUTION_STREAM_SEQUENCE
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/sequence.cuh>
|
||||
#include <cuda/experimental/__execution/stream/domain.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
// /////////////////////////////////////////////////////////////////////////////////
|
||||
// // sequence: customization for the stream scheduler
|
||||
// template <>
|
||||
// struct stream_domain::__apply_t<sequence_t>
|
||||
// {
|
||||
// template <class _Sndr, class _Env>
|
||||
// _CCCL_HOST_DEVICE_API auto operator()(_Sndr __sndr, const _Env& __env) const
|
||||
// {
|
||||
// static_assert(::cuda::std::__always_false_v<_Sndr>,
|
||||
// "The CUDA stream scheduler does not yet support the 'sequence' algorithm.");
|
||||
// }
|
||||
// };
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_STREAM_SEQUENCE
|
||||
@@ -1,130 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_STREAM_SYNC_WAIT
|
||||
#define __CUDAX_EXECUTION_STREAM_SYNC_WAIT
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__utility/move.h>
|
||||
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/stream/domain.cuh>
|
||||
#include <cuda/experimental/__execution/sync_wait.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
namespace __stream
|
||||
{
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
// sync_wait: customization for the stream scheduler
|
||||
struct __sync_wait_t : private sync_wait_t
|
||||
{
|
||||
// TODO: calling sync_wait from device code is not supported yet.
|
||||
template <class _Sndr, class _Env>
|
||||
_CCCL_HOST_DEVICE_API auto operator()(_Sndr&& __sndr, _Env&& __env) const
|
||||
{
|
||||
// _Sndr is a sender that has not yet been transformed to run on the stream domain.
|
||||
// The transformation would happen in due course in the connect cpo, so why transform
|
||||
// it here? This transformation shuffles the sender into one that can provide a
|
||||
// stream_ref, which is needed by __host_apply.
|
||||
auto __new_sndr = stream_domain{}.transform_sender(set_value, static_cast<_Sndr&&>(__sndr), __env);
|
||||
|
||||
NV_IF_ELSE_TARGET(NV_IS_HOST,
|
||||
(return __host_apply(_CCCL_MOVE(__new_sndr), static_cast<_Env&&>(__env));),
|
||||
(return __device_apply(_CCCL_MOVE(__new_sndr), static_cast<_Env&&>(__env));))
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API auto operator()(_Sndr&& __sndr) const
|
||||
{
|
||||
return (*this)(static_cast<_Sndr&&>(__sndr), env{});
|
||||
}
|
||||
|
||||
private:
|
||||
template <class _Sndr, class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __managed_state_t
|
||||
{
|
||||
using __partial_completions_t = completion_signatures_of_t<_Sndr, __env_t<_Env>>;
|
||||
using __all_nothrow_t =
|
||||
typename __partial_completions_t::template __transform_q<__nothrow_decay_copyable_t, ::cuda::std::_And>;
|
||||
|
||||
using __completions_t =
|
||||
__concat_completion_signatures_t<__partial_completions_t, __eptr_completion_if_t<!__all_nothrow_t::value>>;
|
||||
|
||||
using __values_t = __value_types<__completions_t, __decayed_tuple, ::cuda::std::__type_self_t>;
|
||||
using __errors_t = __error_types<__completions_t, __decayed_variant>;
|
||||
using __rcvr_t = sync_wait_t::__rcvr_t<__values_t, __errors_t, _Env>;
|
||||
|
||||
_CCCL_HOST_API explicit __managed_state_t(_Sndr&& __sndr, _Env&& __env)
|
||||
: __result_{}
|
||||
, __state_{static_cast<_Env&&>(__env), &__result_}
|
||||
, __opstate_{execution::connect(static_cast<_Sndr&&>(__sndr), __rcvr_t{&__state_})}
|
||||
{}
|
||||
|
||||
::cuda::std::optional<__values_t> __result_;
|
||||
sync_wait_t::__state_t<__values_t, __errors_t, _Env> __state_;
|
||||
connect_result_t<_Sndr, __rcvr_t> __opstate_;
|
||||
};
|
||||
|
||||
template <class _Sndr, class _Env>
|
||||
_CCCL_DEVICE_API static auto __device_apply(_Sndr&& __sndr, _Env&& __env)
|
||||
{
|
||||
return sync_wait.apply_sender(static_cast<_Sndr&&>(__sndr), static_cast<_Env&&>(__env));
|
||||
}
|
||||
|
||||
template <class _Sndr, class _Env>
|
||||
_CCCL_HOST_API static auto __host_apply(_Sndr&& __sndr, _Env&& __env)
|
||||
{
|
||||
stream_ref __stream = __get_stream(__sndr, __env);
|
||||
|
||||
// Launch the sender with a continuation that will fill in a variant
|
||||
using __box_t = __managed_box<__managed_state_t<_Sndr, _Env>>;
|
||||
auto __box = __box_t::__make_unique(static_cast<_Sndr&&>(__sndr), static_cast<_Env&&>(__env));
|
||||
execution::start(__box->__value.__opstate_);
|
||||
|
||||
// The kernels have been launched, now we sync the stream to guarantee forward progress.
|
||||
__stream.sync();
|
||||
|
||||
// While waiting for the variant to be filled in, process any work that may be
|
||||
// delegated to this thread.
|
||||
auto& __state = __box->__value.__state_;
|
||||
__state.__loop_.run();
|
||||
|
||||
if (__state.__errors_.__index() != __npos)
|
||||
{
|
||||
__visit(sync_wait_t::__throw_error_fn{}, _CCCL_MOVE(__state.__errors_));
|
||||
}
|
||||
|
||||
return _CCCL_MOVE(__box->__value.__result_);
|
||||
}
|
||||
};
|
||||
} // namespace __stream
|
||||
|
||||
template <>
|
||||
struct stream_domain::__apply_t<sync_wait_t> : __stream::__sync_wait_t
|
||||
{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_STREAM_SYNC_WAIT
|
||||
@@ -1,36 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_STREAM_CONTEXT
|
||||
#define __CUDAX_EXECUTION_STREAM_CONTEXT
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
// IWYU pragma: begin_exports
|
||||
#include <cuda/experimental/__execution/stream/adaptor.cuh>
|
||||
#include <cuda/experimental/__execution/stream/bulk.cuh>
|
||||
#include <cuda/experimental/__execution/stream/context.cuh>
|
||||
#include <cuda/experimental/__execution/stream/domain.cuh>
|
||||
#include <cuda/experimental/__execution/stream/launch.cuh>
|
||||
#include <cuda/experimental/__execution/stream/let_value.cuh>
|
||||
#include <cuda/experimental/__execution/stream/schedule_from.cuh>
|
||||
#include <cuda/experimental/__execution/stream/sequence.cuh>
|
||||
#include <cuda/experimental/__execution/stream/sync_wait.cuh>
|
||||
// IWYU pragma: end_exports
|
||||
|
||||
#endif //__CUDAX_EXECUTION_STREAM_CONTEXT
|
||||
@@ -1,331 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_SYNC_WAIT
|
||||
#define __CUDAX_EXECUTION_SYNC_WAIT
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__exception/cuda_error.h>
|
||||
#include <cuda/std/__exception/exception_macros.h>
|
||||
#include <cuda/std/__type_traits/always_false.h>
|
||||
#include <cuda/std/__type_traits/conjunction.h>
|
||||
#include <cuda/std/__type_traits/decay.h>
|
||||
#include <cuda/std/optional>
|
||||
#include <cuda/std/tuple>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/apply_sender.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/exception.cuh>
|
||||
#include <cuda/experimental/__execution/meta.cuh>
|
||||
#include <cuda/experimental/__execution/run_loop.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__execution/variant.cuh>
|
||||
#include <cuda/experimental/__execution/write_env.cuh>
|
||||
|
||||
#include <exception>
|
||||
#include <system_error>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
/// @brief Function object type for synchronously waiting for the result of a
|
||||
/// sender.
|
||||
struct sync_wait_t
|
||||
{
|
||||
_CUDAX_SEMI_PRIVATE :
|
||||
template <class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __state_base_t
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __state_base_t(_Env __env) noexcept
|
||||
: __loop_(static_cast<_Env&&>(__env))
|
||||
{}
|
||||
|
||||
// FUTURE: if _Env provides a delegation scheduler, we don't need the run_loop (?)
|
||||
basic_run_loop<_Env> __loop_;
|
||||
};
|
||||
|
||||
public:
|
||||
template <class _Env = env<>>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __env_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Query, class... _Args)
|
||||
_CCCL_REQUIRES(__queryable_with<_Env, _Query, _Args...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(_Query, _Args&&... __args) const
|
||||
noexcept(__nothrow_queryable_with<_Env, _Query, _Args...>) -> __query_result_t<_Env, _Query, _Args...>
|
||||
{
|
||||
return get_env(__state_->__loop_).query(_Query{}, static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_scheduler_t) const noexcept
|
||||
{
|
||||
if constexpr (__queryable_with<_Env, get_scheduler_t>)
|
||||
{
|
||||
return get_env(__state_->__loop_).query(get_scheduler);
|
||||
}
|
||||
else
|
||||
{
|
||||
return __state_->__loop_.get_scheduler();
|
||||
}
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_delegation_scheduler_t) const noexcept
|
||||
{
|
||||
if constexpr (__queryable_with<_Env, get_delegation_scheduler_t>)
|
||||
{
|
||||
return get_env(__state_->__loop_).query(get_delegation_scheduler);
|
||||
}
|
||||
else
|
||||
{
|
||||
return __state_->__loop_.get_scheduler();
|
||||
}
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
__state_base_t<_Env>* __state_;
|
||||
};
|
||||
|
||||
template <class... _Ts>
|
||||
using __decayed_tuple = ::cuda::std::tuple<decay_t<_Ts>...>;
|
||||
|
||||
_CUDAX_SEMI_PRIVATE :
|
||||
template <class _Values, class _Errors, class _Env = env<>>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __state_t : __state_base_t<_Env>
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __state_t(_Env __env, ::cuda::std::optional<_Values>* __values) noexcept
|
||||
: __state_base_t<_Env>{static_cast<_Env&&>(__env)}
|
||||
, __values_{__values}
|
||||
{}
|
||||
|
||||
::cuda::std::optional<_Values>* __values_{};
|
||||
_Errors __errors_{};
|
||||
};
|
||||
|
||||
template <class _Values, class _Errors, class _Env = env<>>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_t
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
template <class... _As>
|
||||
_CCCL_HOST_DEVICE_API void set_value(_As&&... __as) noexcept
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
__state_->__values_->emplace(static_cast<_As&&>(__as)...);
|
||||
}
|
||||
_CCCL_CATCH_ALL
|
||||
{
|
||||
// avoid ODR-using a call to __emplace(exception_ptr) if this code is unreachable.
|
||||
if constexpr (!__nothrow_decay_copyable<_As...>)
|
||||
{
|
||||
__state_->__errors_.__emplace(execution::current_exception());
|
||||
}
|
||||
}
|
||||
__state_->__loop_.finish();
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API void set_error(_Error&& __err) noexcept
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
__state_->__errors_.__emplace(static_cast<_Error&&>(__err));
|
||||
}
|
||||
_CCCL_CATCH_ALL
|
||||
{
|
||||
// avoid ODR-using a call to __emplace(exception_ptr) if this code is unreachable.
|
||||
if constexpr (!__nothrow_decay_copyable<_Error>)
|
||||
{
|
||||
__state_->__errors_.__emplace(execution::current_exception());
|
||||
}
|
||||
}
|
||||
__state_->__loop_.finish();
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_stopped() noexcept
|
||||
{
|
||||
__state_->__loop_.finish();
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __env_t<_Env>
|
||||
{
|
||||
return __env_t<_Env>{__state_};
|
||||
}
|
||||
|
||||
__state_t<_Values, _Errors, _Env>* __state_;
|
||||
};
|
||||
|
||||
struct __throw_error_fn
|
||||
{
|
||||
template <class _Error>
|
||||
[[noreturn]]
|
||||
_CCCL_HOST_DEVICE_API void operator()(_Error __err) const
|
||||
{
|
||||
NV_IF_ELSE_TARGET(NV_IS_HOST, (__do_throw(static_cast<_Error&&>(__err));), (::cuda::std::terminate();))
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
[[noreturn]]
|
||||
_CCCL_HOST_API static void __do_throw(_Error __err)
|
||||
{
|
||||
if constexpr (__same_as<_Error, exception_ptr>)
|
||||
{
|
||||
execution::rethrow_exception(static_cast<_Error&&>(__err));
|
||||
}
|
||||
else if constexpr (__same_as<_Error, ::std::exception_ptr>)
|
||||
{
|
||||
::std::rethrow_exception(static_cast<_Error&&>(__err));
|
||||
}
|
||||
else if constexpr (__same_as<_Error, ::std::error_code>)
|
||||
{
|
||||
throw ::std::system_error(__err);
|
||||
}
|
||||
else if constexpr (__same_as<_Error, cudaError_t>)
|
||||
{
|
||||
_CCCL_THROW(::cuda::cuda_error, __err, "sync_wait failed with cudaError_t");
|
||||
}
|
||||
else
|
||||
{
|
||||
throw static_cast<_Error&&>(__err);
|
||||
}
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Diagnostic>
|
||||
struct __bad_sync_wait
|
||||
{
|
||||
static_assert(::cuda::std::__always_false_v<_Diagnostic>,
|
||||
"sync_wait cannot compute the completions of the sender passed to it.");
|
||||
_CCCL_HOST_API static auto __result() -> __bad_sync_wait;
|
||||
|
||||
_CCCL_HOST_API auto value() const -> const __bad_sync_wait&;
|
||||
_CCCL_HOST_API auto operator*() const -> const __bad_sync_wait&;
|
||||
|
||||
// Attempt to suppress follow-on errors about non-convertibility after the one already
|
||||
// reported.
|
||||
template <class _Ty>
|
||||
_CCCL_HOST_DEVICE_API operator _Ty&&() const noexcept;
|
||||
|
||||
int i{}; // so that structured bindings kinda work
|
||||
};
|
||||
|
||||
public:
|
||||
// This is the actual default sync_wait implementation.
|
||||
template <class _Sndr, class _Env>
|
||||
_CCCL_HOST_DEVICE_API static auto apply_sender(_Sndr&& __sndr, _Env&& __env)
|
||||
{
|
||||
using __partial_completions_t = completion_signatures_of_t<_Sndr, __env_t<_Env>>;
|
||||
using __all_nothrow_t =
|
||||
typename __partial_completions_t::template __transform_q<__nothrow_decay_copyable_t, ::cuda::std::_And>;
|
||||
|
||||
using __completions_t =
|
||||
__concat_completion_signatures_t<__partial_completions_t, __eptr_completion_if_t<!__all_nothrow_t::value>>;
|
||||
|
||||
using __values_t = __value_types<__completions_t, __decayed_tuple, ::cuda::std::__type_self_t>;
|
||||
using __errors_t = __error_types<__completions_t, __decayed_variant>;
|
||||
|
||||
::cuda::std::optional<__values_t> __result{};
|
||||
__state_t<__values_t, __errors_t, _Env> __state(static_cast<_Env&&>(__env), &__result);
|
||||
|
||||
// Launch the sender with a continuation that will fill in a variant
|
||||
auto __opstate = execution::connect(static_cast<_Sndr&&>(__sndr), __rcvr_t<__values_t, __errors_t, _Env>{&__state});
|
||||
execution::start(__opstate);
|
||||
|
||||
// While waiting for the variant to be filled in, process any work that may be
|
||||
// delegated to this thread.
|
||||
__state.__loop_.run();
|
||||
|
||||
if (__state.__errors_.__index() != __npos)
|
||||
{
|
||||
__visit(__throw_error_fn{}, static_cast<__errors_t&&>(__state.__errors_));
|
||||
}
|
||||
|
||||
return __result; // uses NRVO to return the result
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API static auto apply_sender(_Sndr&& __sndr)
|
||||
{
|
||||
return apply_sender(static_cast<_Sndr&&>(__sndr), env{});
|
||||
}
|
||||
|
||||
// clang-format off
|
||||
/// @brief Synchronously wait for the result of a sender, blocking the
|
||||
/// current thread.
|
||||
///
|
||||
/// `sync_wait` connects and starts the given sender, and then drives a
|
||||
/// `run_loop` instance until the sender completes. Additional work
|
||||
/// can be delegated to the `run_loop` by scheduling work on the
|
||||
/// scheduler returned by calling `get_delegation_scheduler` on the
|
||||
/// receiver's environment.
|
||||
///
|
||||
/// @pre The sender must have a exactly one value completion signature. That
|
||||
/// is, it can only complete successfully in one way, with a single
|
||||
/// set of values.
|
||||
///
|
||||
/// @retval success Returns an engaged `cuda::std::optional` containing the result
|
||||
/// values in a `cuda::std::tuple`.
|
||||
/// @retval canceled Returns an empty `cuda::std::optional`.
|
||||
/// @retval error Throws the error.
|
||||
///
|
||||
/// @throws ::std::rethrow_exception(error) if the error has type
|
||||
/// `exception_ptr`.
|
||||
/// @throws ::std::system_error(error) if the error has type
|
||||
/// `::std::error_code`.
|
||||
/// @throws ::cuda::cuda_error(error, "...") if the error has type
|
||||
/// `cudaError_t`.
|
||||
/// @throws error otherwise
|
||||
// clang-format on
|
||||
template <class _Sndr, class _Env = env<>>
|
||||
_CCCL_HOST_DEVICE_API auto operator()(_Sndr&& __sndr, _Env&& __env = {}) const
|
||||
{
|
||||
using __env_t = sync_wait_t::__env_t<_Env>;
|
||||
constexpr auto __completions = get_completion_signatures<_Sndr, __env_t>();
|
||||
using __completions_t = decltype(__completions);
|
||||
|
||||
if constexpr (!__valid_completion_signatures<__completions_t>)
|
||||
{
|
||||
return __bad_sync_wait<__completions_t>::__result();
|
||||
}
|
||||
else if constexpr (__completions.count(set_value) != 1)
|
||||
{
|
||||
static_assert(__completions.count(set_value) == 1,
|
||||
"sync_wait requires a sender with exactly one value completion signature.");
|
||||
}
|
||||
else
|
||||
{
|
||||
constexpr auto __domain = __completion_domain_of_t<set_value_t, _Sndr, __env_t>();
|
||||
return execution::apply_sender(__domain, *this, static_cast<_Sndr&&>(__sndr), static_cast<_Env&&>(__env));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT sync_wait_t sync_wait{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_SYNC_WAIT
|
||||
@@ -1,718 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_TASK_SCHEDULER
|
||||
#define __CUDAX_EXECUTION_TASK_SCHEDULER
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__exception/cuda_error.h>
|
||||
#include <cuda/std/__exception/terminate.h>
|
||||
#include <cuda/std/__memory/addressof.h>
|
||||
#include <cuda/std/__memory/allocator.h>
|
||||
#include <cuda/std/__tuple_dir/ignore.h>
|
||||
#include <cuda/std/__utility/pod_tuple.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/bulk.cuh>
|
||||
#include <cuda/experimental/__execution/concepts.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/diagnostics.cuh>
|
||||
#include <cuda/experimental/__execution/domain.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/exception.cuh>
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/inline_scheduler.cuh>
|
||||
#include <cuda/experimental/__execution/parallel_scheduler_backend.cuh>
|
||||
#include <cuda/experimental/__execution/rcvr_ref.cuh>
|
||||
#include <cuda/experimental/__execution/transform_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/variant.cuh>
|
||||
#include <cuda/experimental/__utility/shared_ptr.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
struct task_scheduler;
|
||||
|
||||
struct task_scheduler_domain;
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
// The concrete type-erased sender returned by task_scheduler::schedule()
|
||||
struct __task_sender;
|
||||
|
||||
template <class _Sndr>
|
||||
struct __task_bulk_sender;
|
||||
|
||||
template <class _BulkTag, class _Policy, class _Fn, class _Rcvr, class _Values>
|
||||
class __task_bulk_state;
|
||||
|
||||
template <class _BulkTag, class _Policy, class _Fn, class _Rcvr, class _Values>
|
||||
struct __task_bulk_receiver;
|
||||
|
||||
struct __task_scheduler_backend : parallel_scheduler_backend
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API virtual auto query(get_forward_progress_guarantee_t) const noexcept
|
||||
-> forward_progress_guarantee = 0;
|
||||
_CCCL_HOST_DEVICE_API virtual auto __equal_to(const void* __other, ::cuda::std::__type_info_ref __type) -> bool = 0;
|
||||
};
|
||||
|
||||
using __backend_ptr_t = __shared_ptr<__task_scheduler_backend>;
|
||||
|
||||
template <class _Sch>
|
||||
_CCCL_CONCEPT __non_task_scheduler = _CCCL_REQUIRES_EXPR((_Sch))( //
|
||||
requires(__not_same_as<task_scheduler, _Sch>), //
|
||||
requires(scheduler<_Sch>));
|
||||
} // namespace __detail
|
||||
|
||||
struct _CANNOT_DISPATCH_BULK_ALGORITHM_TO_TASK_SCHEDULER_BECAUSE_THERE_IS_NO_TASK_SCHEDULER_IN_THE_ENVIRONMENT;
|
||||
struct _ADD_A_CONTINUES_ON_TRANSITION_TO_THE_TASK_SCHEDULER_BEFORE_THE_BULK_ALGORITHM;
|
||||
|
||||
struct task_scheduler_domain : default_domain
|
||||
{
|
||||
_CCCL_TEMPLATE(class _Sndr, class _Env, class _BulkTag = tag_of_t<_Sndr>)
|
||||
_CCCL_REQUIRES(__one_of<_BulkTag, bulk_chunked_t, bulk_unchunked_t>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto
|
||||
transform_sender(set_value_t, _Sndr&& __sndr, const _Env& __env)
|
||||
{
|
||||
using __sched_t =
|
||||
__call_result_or_t<get_completion_scheduler_t<set_value_t>, __not_a_scheduler<>, env_of_t<_Sndr>, const _Env&>;
|
||||
if constexpr (!__same_as<__sched_t, task_scheduler>)
|
||||
{
|
||||
return __not_a_sender<
|
||||
_WHERE(_IN_ALGORITHM, _BulkTag),
|
||||
_WHAT(_CANNOT_DISPATCH_BULK_ALGORITHM_TO_TASK_SCHEDULER_BECAUSE_THERE_IS_NO_TASK_SCHEDULER_IN_THE_ENVIRONMENT),
|
||||
_TO_FIX_THIS_ERROR(_ADD_A_CONTINUES_ON_TRANSITION_TO_THE_TASK_SCHEDULER_BEFORE_THE_BULK_ALGORITHM),
|
||||
_WITH_SENDER(_Sndr),
|
||||
_WITH_ENVIRONMENT(_Env)>{};
|
||||
}
|
||||
else
|
||||
{
|
||||
auto __sch = get_completion_scheduler<set_value_t>(get_env(__sndr), __env);
|
||||
return __detail::__task_bulk_sender<_Sndr>{static_cast<_Sndr&&>(__sndr), _CCCL_MOVE(__sch)};
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
//! @brief A type-erased scheduler.
|
||||
//!
|
||||
//! The `task_scheduler` struct is implemented in terms of a backend type derived from
|
||||
//! @c parallel_scheduler_backend, providing a type-erased interface for scheduling tasks.
|
||||
//! It exposes query functions to retrieve the completion scheduler and domain.
|
||||
//!
|
||||
//! @note This scheduler is designed for use with CUDA experimental execution APIs.
|
||||
//!
|
||||
//! @see parallel_scheduler_backend
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT task_scheduler
|
||||
{
|
||||
template <class _Sch, class _Alloc>
|
||||
class _CCCL_TYPE_VISIBILITY_DEFAULT __backend_for;
|
||||
|
||||
public:
|
||||
using scheduler_concept = scheduler_t;
|
||||
|
||||
_CCCL_TEMPLATE(class _Sch, class _Alloc = ::cuda::std::allocator<::cuda::std::byte>)
|
||||
_CCCL_REQUIRES(__detail::__non_task_scheduler<_Sch>)
|
||||
_CCCL_HOST_DEVICE_API explicit task_scheduler(_Sch __sch, _Alloc __alloc = {})
|
||||
: __backend_(experimental::__allocate_shared<__backend_for<_Sch, _Alloc>>(__alloc, _CCCL_MOVE(__sch), __alloc))
|
||||
{}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto schedule() const noexcept -> __detail::__task_sender;
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend bool
|
||||
operator==(const task_scheduler& __lhs, const task_scheduler& __rhs) noexcept
|
||||
{
|
||||
return __lhs.__backend_ == __rhs.__backend_;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend bool
|
||||
operator!=(const task_scheduler& __lhs, const task_scheduler& __rhs) noexcept
|
||||
{
|
||||
return !(__lhs.__backend_ == __rhs.__backend_);
|
||||
}
|
||||
|
||||
template <class _Sch>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend auto operator==(const task_scheduler& __lhs, const _Sch& __rhs) noexcept
|
||||
_CCCL_TRAILING_REQUIRES(bool)(__detail::__non_task_scheduler<_Sch>)
|
||||
{
|
||||
return __lhs.__backend_->__equal_to(::cuda::std::addressof(__rhs), _CCCL_TYPEID(_Sch));
|
||||
}
|
||||
|
||||
template <class _Sch>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend auto operator!=(const task_scheduler& __lhs, const _Sch& __rhs) noexcept
|
||||
_CCCL_TRAILING_REQUIRES(bool)(__detail::__non_task_scheduler<_Sch>)
|
||||
{
|
||||
return !(__lhs == __rhs);
|
||||
}
|
||||
|
||||
template <class _Sch>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend auto operator==(const _Sch& __lhs, const task_scheduler& __rhs) noexcept
|
||||
_CCCL_TRAILING_REQUIRES(bool)(__detail::__non_task_scheduler<_Sch>)
|
||||
{
|
||||
return __rhs == __lhs;
|
||||
}
|
||||
|
||||
template <class _Sch>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend auto operator!=(const _Sch& __lhs, const task_scheduler& __rhs) noexcept
|
||||
_CCCL_TRAILING_REQUIRES(bool)(__detail::__non_task_scheduler<_Sch>)
|
||||
{
|
||||
return !(__rhs == __lhs);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto query(get_forward_progress_guarantee_t) const noexcept
|
||||
-> forward_progress_guarantee
|
||||
{
|
||||
return __backend_->query(get_forward_progress_guarantee_t{});
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto query(get_completion_scheduler_t<set_value_t>) const noexcept
|
||||
-> const task_scheduler&
|
||||
{
|
||||
return *this;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_domain_t<set_value_t>) const noexcept
|
||||
{
|
||||
return task_scheduler_domain{};
|
||||
}
|
||||
|
||||
private:
|
||||
template <class>
|
||||
friend struct __detail::__task_bulk_sender;
|
||||
friend struct __detail::__task_sender;
|
||||
|
||||
__detail::__backend_ptr_t __backend_;
|
||||
};
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
//! @brief A type-erased opstate returned when connecting the result of
|
||||
//! task_scheduler::schedule() to a receiver.
|
||||
template <class _Rcvr>
|
||||
class __task_opstate_t
|
||||
{
|
||||
public:
|
||||
using operation_state_concept = operation_state_t;
|
||||
|
||||
_CCCL_HOST_DEVICE_API __task_opstate_t(__backend_ptr_t __backend, _Rcvr __rcvr)
|
||||
: __rcvr_proxy_(_CCCL_MOVE(__rcvr))
|
||||
, __backend_(_CCCL_MOVE(__backend))
|
||||
{}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void start() noexcept
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
__backend_->schedule(__rcvr_proxy_, ::cuda::std::span{__storage_});
|
||||
}
|
||||
_CCCL_CATCH_ALL
|
||||
{
|
||||
__rcvr_proxy_.set_error(execution::current_exception());
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
__detail::__receiver_proxy<_Rcvr> __rcvr_proxy_;
|
||||
__backend_ptr_t __backend_;
|
||||
::cuda::std::byte __storage_[8 * sizeof(void*)];
|
||||
};
|
||||
|
||||
//! @brief A type-erased sender returned by task_scheduler::schedule().
|
||||
struct __task_sender
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
using __completions_t =
|
||||
completion_signatures<set_value_t(), //
|
||||
set_error_t(exception_ptr),
|
||||
set_error_t(cudaError_t),
|
||||
set_stopped_t()>;
|
||||
|
||||
_CCCL_HOST_DEVICE_API explicit __task_sender(task_scheduler __sch)
|
||||
: __attrs_{_CCCL_MOVE(__sch)}
|
||||
{}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto connect(_Rcvr __rcvr) const noexcept -> __task_opstate_t<_Rcvr>
|
||||
{
|
||||
return __task_opstate_t<_Rcvr>(get_completion_scheduler<set_value_t>(__attrs_).__backend_, _CCCL_MOVE(__rcvr));
|
||||
}
|
||||
|
||||
template <class _Self>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures() noexcept
|
||||
-> __completions_t
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto get_env() const noexcept -> const __sch_attrs_t<task_scheduler>&
|
||||
{
|
||||
return __attrs_;
|
||||
}
|
||||
|
||||
private:
|
||||
__sch_attrs_t<task_scheduler> __attrs_;
|
||||
};
|
||||
|
||||
//! @brief A receiver used to connect the predecessor of a bulk operation launched by a
|
||||
//! task_scheduler. Its set_value member stores the predecessor's values in the bulk
|
||||
//! operation state and then starts the bulk operation.
|
||||
template <class _BulkTag, class _Policy, class _Fn, class _Rcvr, class _Values>
|
||||
struct __task_bulk_receiver
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
template <class... _As>
|
||||
_CCCL_HOST_DEVICE_API void set_value(_As&&... __as) noexcept
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
// Store the predecessor's values in the bulk operation state.
|
||||
using __values_t = ::cuda::std::__decayed_tuple<_As...>;
|
||||
__state_->__values_.template __emplace<__values_t>(static_cast<_As&&>(__as)...);
|
||||
|
||||
// Start the bulk operation.
|
||||
if constexpr (__same_as<_BulkTag, bulk_chunked_t>)
|
||||
{
|
||||
__state_->__backend_->schedule_bulk_chunked(
|
||||
__state_->__shape_, *__state_, ::cuda::std::span{__state_->__storage_});
|
||||
}
|
||||
else
|
||||
{
|
||||
__state_->__backend_->schedule_bulk_unchunked(
|
||||
__state_->__shape_, *__state_, ::cuda::std::span{__state_->__storage_});
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_ALL
|
||||
{
|
||||
execution::set_error(_CCCL_MOVE(__state_->__rcvr_), execution::current_exception());
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API void set_error(_Error&& __err) noexcept
|
||||
{
|
||||
execution::set_error(_CCCL_MOVE(__state_->__rcvr_), static_cast<_Error&&>(__err));
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void set_stopped() noexcept
|
||||
{
|
||||
execution::set_stopped(_CCCL_MOVE(__state_->__rcvr_));
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto get_env() const noexcept -> env_of_t<_Rcvr>
|
||||
{
|
||||
return execution::get_env(__state_->__rcvr_);
|
||||
}
|
||||
|
||||
__task_bulk_state<_BulkTag, _Policy, _Fn, _Rcvr, _Values>* __state_;
|
||||
};
|
||||
|
||||
//! Returns a visitor (callable) used to invoke the bulk (unchunked) function with the
|
||||
//! predecessor's values, which are stored in a variant in the bulk operation state.
|
||||
template <bool _Parallelize, class _Fn>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
__get_execute_bulk_fn(bulk_unchunked_t, _Fn& __fn, size_t __shape, size_t __begin, size_t) noexcept
|
||||
{
|
||||
return [=, &__fn](auto& __args) {
|
||||
constexpr bool __valid_args = !__same_as<decltype(__args), ::cuda::std::monostate&>;
|
||||
// runtime assert that we never take this path without valid args from the predecessor:
|
||||
_CCCL_ASSERT(__valid_args, "internal error: predecessor results are not stored in the bulk operation state");
|
||||
|
||||
if constexpr (__valid_args)
|
||||
{
|
||||
// If we are not parallelizing, we need to run all the iterations sequentially.
|
||||
const size_t __increments = _Parallelize ? 1 : __shape;
|
||||
// Precompose the function with the arguments so we don't have to do it every iteration.
|
||||
auto __precomposed_fn = ::cuda::std::__apply(
|
||||
[&](auto&... __as) {
|
||||
return [&](size_t __i) -> void {
|
||||
__fn(__i, __as...);
|
||||
};
|
||||
},
|
||||
__args);
|
||||
for (size_t __i = __begin; __i < __begin + __increments; ++__i)
|
||||
{
|
||||
__precomposed_fn(__i);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
template <bool _Parallelize, class _Fn>
|
||||
struct __apply_bulk_execute
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class... _As>
|
||||
_CCCL_HOST_DEVICE_API void operator()(_As&... __as) const noexcept(__nothrow_callable<_Fn&, size_t, _As&...>)
|
||||
{
|
||||
if constexpr (_Parallelize)
|
||||
{
|
||||
__fn_(__begin_, __end_, __as...);
|
||||
}
|
||||
else
|
||||
{
|
||||
// If we are not parallelizing, we need to pass the entire range to the functor.
|
||||
__fn_(size_t(0), __shape_, __as...);
|
||||
}
|
||||
}
|
||||
|
||||
size_t __begin_, __end_, __shape_;
|
||||
_Fn& __fn_;
|
||||
};
|
||||
|
||||
//! Returns a visitor (callable) used to invoke the bulk (chunked) function with the
|
||||
//! predecessor's values, which are stored in a variant in the bulk operation state.
|
||||
template <bool _Parallelize, class _Fn>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
__get_execute_bulk_fn(bulk_chunked_t, _Fn& __fn, size_t __shape, size_t __begin, size_t __end) noexcept
|
||||
{
|
||||
return [=, &__fn](auto& __args) {
|
||||
constexpr bool __valid_args = !__same_as<decltype(__args), ::cuda::std::monostate&>;
|
||||
_CCCL_ASSERT(__valid_args, "internal error: predecessor results are not stored in the bulk operation state");
|
||||
|
||||
if constexpr (__valid_args)
|
||||
{
|
||||
::cuda::std::__apply(__apply_bulk_execute<_Parallelize, _Fn>{__begin, __end, __shape, __fn}, __args);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
//! Stores the state for a bulk operation launched by a task_scheduler. A type-erased
|
||||
//! reference to this object is passed to either the task_scheduler's
|
||||
//! schedule_bulk_chunked or schedule_bulk_unchunked methods, which is expected to call
|
||||
//! execute(begin, end) on it to run the bulk operation. After the bulk operation is
|
||||
//! complete, set_value is called, which forwards the predecessor's values to the
|
||||
//! downstream receiver.
|
||||
template <class _BulkTag, class _Policy, class _Fn, class _Rcvr, class _Values>
|
||||
class __task_bulk_state : public __detail::__receiver_proxy_base<_Rcvr, bulk_item_receiver_proxy>
|
||||
{
|
||||
public:
|
||||
_CCCL_HOST_DEVICE_API explicit __task_bulk_state(_Rcvr __rcvr, size_t __shape, _Fn __fn, __backend_ptr_t __backend)
|
||||
: __task_bulk_state::__receiver_proxy_base(_CCCL_MOVE(__rcvr))
|
||||
, __fn_(_CCCL_MOVE(__fn))
|
||||
, __shape_(__shape)
|
||||
, __backend_(_CCCL_MOVE(__backend))
|
||||
{}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void set_value() noexcept final override
|
||||
{
|
||||
// Send the stored values to the downstream receiver.
|
||||
__visit(
|
||||
[this](auto& __tupl) {
|
||||
constexpr bool __valid_args = __not_same_as<decltype(__tupl), ::cuda::std::monostate&>;
|
||||
// runtime assert that we never take this path without valid args from the predecessor:
|
||||
_CCCL_ASSERT(__valid_args, "internal error: predecessor results are not stored in the bulk operation state");
|
||||
|
||||
if constexpr (__valid_args)
|
||||
{
|
||||
::cuda::std::__apply(execution::set_value, _CCCL_MOVE(__tupl), _CCCL_MOVE(this->__rcvr_));
|
||||
}
|
||||
},
|
||||
__values_);
|
||||
}
|
||||
|
||||
//! Actually runs the bulk operation over the specified range.
|
||||
_CCCL_HOST_DEVICE_API void execute(size_t __begin, size_t __end) noexcept final override
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
constexpr bool __parallelize =
|
||||
::cuda::std::is_same_v<_Policy, ::cuda::std::execution::parallel_policy>
|
||||
|| ::cuda::std::is_same_v<_Policy, ::cuda::std::execution::parallel_unsequenced_policy>;
|
||||
__visit(__detail::__get_execute_bulk_fn<__parallelize>(_BulkTag(), __fn_, __shape_, __begin, __end), __values_);
|
||||
}
|
||||
_CCCL_CATCH_ALL
|
||||
{
|
||||
execution::set_error(_CCCL_MOVE(this->__rcvr_), execution::current_exception());
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
template <class, class, class, class, class>
|
||||
friend struct __task_bulk_receiver;
|
||||
|
||||
_Fn __fn_;
|
||||
size_t __shape_;
|
||||
_Values __values_{};
|
||||
__backend_ptr_t __backend_;
|
||||
::cuda::std::byte __storage_[8 * sizeof(void*)];
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////
|
||||
// Operation state for task scheduler bulk operations
|
||||
template <class _BulkTag, class _Policy, class _Sndr, class _Fn, class _Rcvr>
|
||||
struct __task_bulk_opstate
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
|
||||
_CCCL_HOST_DEVICE_API explicit __task_bulk_opstate(
|
||||
_Sndr&& __sndr, size_t __shape, _Fn __fn, _Rcvr __rcvr, __backend_ptr_t __backend)
|
||||
: __state_{_CCCL_MOVE(__rcvr), __shape, _CCCL_MOVE(__fn), _CCCL_MOVE(__backend)}
|
||||
, __opstate1_(execution::connect(static_cast<_Sndr&&>(__sndr), __rcvr_t{&__state_}))
|
||||
{}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void start() noexcept
|
||||
{
|
||||
execution::start(__opstate1_);
|
||||
}
|
||||
|
||||
private:
|
||||
using __values_t =
|
||||
value_types_of_t<_Sndr, __fwd_env_t<env_of_t<_Rcvr>>, ::cuda::std::__decayed_tuple, __nullable_variant>;
|
||||
using __rcvr_t = __task_bulk_receiver<_BulkTag, _Policy, _Fn, _Rcvr, __values_t>;
|
||||
using __opstate1_t = connect_result_t<_Sndr, __rcvr_t>;
|
||||
|
||||
__task_bulk_state<_BulkTag, _Policy, _Fn, _Rcvr, __values_t> __state_;
|
||||
__opstate1_t __opstate1_;
|
||||
};
|
||||
|
||||
template <class _Sndr>
|
||||
struct __task_bulk_sender
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
_CCCL_HOST_DEVICE_API explicit __task_bulk_sender(_Sndr __sndr, task_scheduler __sch)
|
||||
: __sndr_(_CCCL_MOVE(__sndr))
|
||||
, __attrs_{_CCCL_MOVE(__sch)}
|
||||
{}
|
||||
|
||||
template <class _Rcvr>
|
||||
_CCCL_HOST_DEVICE_API auto connect(_Rcvr __rcvr) &&
|
||||
{
|
||||
auto& [__tag, __data, __child] = __sndr_;
|
||||
auto& [__pol, __shape, __fn] = __data;
|
||||
return __task_bulk_opstate<decltype(__tag), decltype(__pol), decltype(__child), decltype(__fn), _Rcvr>{
|
||||
_CCCL_MOVE(__child),
|
||||
static_cast<size_t>(__shape),
|
||||
_CCCL_MOVE(__fn),
|
||||
_CCCL_MOVE(__rcvr),
|
||||
_CCCL_MOVE(__attrs_.__sch_.__backend_)};
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Self, class _Env)
|
||||
_CCCL_REQUIRES(__same_as<_Self, __task_bulk_sender>) // accept only rvalues.
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
// This calls get_completion_signatures on the wrapped bulk_[un]chunked sender. We
|
||||
// call it directly instead of using execution::get_completion_signatures to avoid
|
||||
// another trip through transform_sender, which would lead to infinite recursion.
|
||||
auto __completions = decay_t<_Sndr>::template get_completion_signatures<_Sndr, _Env>();
|
||||
return transform_completion_signatures(__completions, __decay_transform<set_value_t>(), {}, {}, __eptr_completion());
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto get_env() const noexcept -> const __sch_attrs_t<task_scheduler>&
|
||||
{
|
||||
return __attrs_;
|
||||
}
|
||||
|
||||
private:
|
||||
_Sndr __sndr_;
|
||||
__sch_attrs_t<task_scheduler> __attrs_;
|
||||
};
|
||||
|
||||
//! Function called by the `bulk_chunked` operation; calls `execute` on the bulk_item_receiver_proxy.
|
||||
struct __bulk_chunked_fn
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API void operator()(size_t __begin, size_t __end) noexcept
|
||||
{
|
||||
__rcvr_.execute(__begin, __end);
|
||||
}
|
||||
|
||||
bulk_item_receiver_proxy& __rcvr_;
|
||||
};
|
||||
|
||||
//! Function called by the `bulk_unchunked` operation; calls `execute` on the bulk_item_receiver_proxy.
|
||||
struct __bulk_unchunked_fn
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API void operator()(size_t __idx) noexcept
|
||||
{
|
||||
__rcvr_.execute(__idx, __idx + 1);
|
||||
}
|
||||
|
||||
bulk_item_receiver_proxy& __rcvr_;
|
||||
};
|
||||
|
||||
template <class _Ty, class _Alloc, class... _Args>
|
||||
_CCCL_HOST_DEVICE_API auto
|
||||
__emplace_into(::cuda::std::span<::cuda::std::byte> __storage, _Alloc& __alloc, _Args&&... __args) -> _Ty&
|
||||
{
|
||||
using __traits_t = ::cuda::std::allocator_traits<__rebind_alloc_t<_Alloc, _Ty>>;
|
||||
__rebind_alloc_t<_Alloc, _Ty> __alloc_copy{__alloc};
|
||||
|
||||
const bool __in_situ = __storage.size() >= sizeof(_Ty);
|
||||
auto* __ty_ptr = __in_situ ? reinterpret_cast<_Ty*>(__storage.data()) : __traits_t::allocate(__alloc_copy, 1);
|
||||
__traits_t::construct(__alloc_copy, __ty_ptr, static_cast<_Args&&>(__args)...);
|
||||
return *::cuda::std::launder(__ty_ptr);
|
||||
}
|
||||
|
||||
template <class _Alloc, class _Sndr>
|
||||
class __opstate_t : _Alloc
|
||||
{
|
||||
public:
|
||||
using allocator_type = _Alloc;
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_HOST_DEVICE_API __opstate_t(_Alloc __alloc, _Sndr __sndr, receiver_proxy& __rcvr_proxy, bool __in_situ)
|
||||
: _Alloc(_CCCL_MOVE(__alloc))
|
||||
, __opstate_(execution::connect(
|
||||
_CCCL_MOVE(__sndr),
|
||||
__detail::__proxy_receiver<receiver_proxy>{
|
||||
__rcvr_proxy, this, __in_situ ? __delete_opstate<true> : __delete_opstate<false>}))
|
||||
{}
|
||||
__opstate_t(__opstate_t&&) = delete;
|
||||
|
||||
_CCCL_HOST_DEVICE_API void start() noexcept
|
||||
{
|
||||
execution::start(__opstate_);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto query(get_allocator_t) const noexcept -> const _Alloc&
|
||||
{
|
||||
return *this;
|
||||
}
|
||||
|
||||
private:
|
||||
template <bool _InSitu>
|
||||
_CCCL_HOST_DEVICE_API static void __delete_opstate(void* __ptr) noexcept
|
||||
{
|
||||
using __traits_t = ::cuda::std::allocator_traits<__rebind_alloc_t<_Alloc, __opstate_t>>;
|
||||
auto* __opstate = static_cast<__opstate_t*>(__ptr);
|
||||
__rebind_alloc_t<_Alloc, __opstate_t> __alloc_copy{get_allocator(*__opstate)};
|
||||
|
||||
__traits_t::destroy(__alloc_copy, __opstate);
|
||||
if constexpr (!_InSitu)
|
||||
{
|
||||
__traits_t::deallocate(__alloc_copy, __opstate, 1);
|
||||
}
|
||||
}
|
||||
|
||||
using __child_opstate_t = connect_result_t<_Sndr, __detail::__proxy_receiver<receiver_proxy>>;
|
||||
__child_opstate_t __opstate_;
|
||||
};
|
||||
} // namespace __detail
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API inline auto task_scheduler::schedule() const noexcept -> __detail::__task_sender
|
||||
{
|
||||
return __detail::__task_sender{*this};
|
||||
}
|
||||
|
||||
template <class _Sch, class _Alloc>
|
||||
class _CCCL_DECLSPEC_EMPTY_BASES task_scheduler::__backend_for
|
||||
: public __detail::__task_scheduler_backend
|
||||
, _Alloc
|
||||
{
|
||||
template <class _RcvrProxy>
|
||||
friend struct __detail::__proxy_receiver;
|
||||
|
||||
template <class _RcvrProxy, class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API void
|
||||
__schedule(_RcvrProxy& __rcvr_proxy, _Sndr&& __sndr, ::cuda::std::span<::cuda::std::byte> __storage) noexcept
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
using __opstate_t = connect_result_t<_Sndr, __detail::__proxy_receiver<_RcvrProxy>>;
|
||||
const bool __in_situ = __storage.size() >= sizeof(__opstate_t);
|
||||
_Alloc& __alloc = *this;
|
||||
auto& __opstate = __detail::__emplace_into<__detail::__opstate_t<_Alloc, _Sndr>>(
|
||||
__storage, __alloc, __alloc, static_cast<_Sndr&&>(__sndr), __rcvr_proxy, __in_situ);
|
||||
execution::start(__opstate);
|
||||
}
|
||||
_CCCL_CATCH_ALL
|
||||
{
|
||||
__rcvr_proxy.set_error(execution::current_exception());
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
_CCCL_HOST_DEVICE_API explicit __backend_for(_Sch __sch, _Alloc __alloc)
|
||||
: _Alloc(_CCCL_MOVE(__alloc))
|
||||
, __sch_(_CCCL_MOVE(__sch))
|
||||
{}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void
|
||||
schedule(receiver_proxy& __rcvr_proxy, ::cuda::std::span<::cuda::std::byte> __storage) noexcept final override
|
||||
{
|
||||
__schedule(__rcvr_proxy, execution::schedule(__sch_), __storage);
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void schedule_bulk_chunked(
|
||||
size_t __size,
|
||||
bulk_item_receiver_proxy& __rcvr_proxy,
|
||||
::cuda::std::span<::cuda::std::byte> __storage) noexcept final override
|
||||
{
|
||||
auto __sndr =
|
||||
execution::bulk_chunked(execution::schedule(__sch_), par, __size, __detail::__bulk_chunked_fn{__rcvr_proxy});
|
||||
__schedule(__rcvr_proxy, _CCCL_MOVE(__sndr), __storage);
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void schedule_bulk_unchunked(
|
||||
size_t __size,
|
||||
bulk_item_receiver_proxy& __rcvr_proxy,
|
||||
::cuda::std::span<::cuda::std::byte> __storage) noexcept override
|
||||
{
|
||||
auto __sndr =
|
||||
execution::bulk_unchunked(execution::schedule(__sch_), par, __size, __detail::__bulk_unchunked_fn{__rcvr_proxy});
|
||||
__schedule(__rcvr_proxy, _CCCL_MOVE(__sndr), __storage);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API auto query(get_forward_progress_guarantee_t) const noexcept
|
||||
-> forward_progress_guarantee final override
|
||||
{
|
||||
return get_forward_progress_guarantee(__sch_);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_DEVICE_API bool __equal_to(const void* __other, ::cuda::std::__type_info_ref __type) final override
|
||||
{
|
||||
if (__type == _CCCL_TYPEID(_Sch))
|
||||
{
|
||||
const _Sch& __other_sch = *static_cast<const _Sch*>(__other);
|
||||
return __sch_ == __other_sch;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private:
|
||||
_Sch __sch_;
|
||||
};
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
// Implementation of the get_scheduler_t query for __proxy_receiver_impl from
|
||||
// parallel_scheduler_backend.cuh
|
||||
template <class _Rcvr, class _Proxy>
|
||||
_CCCL_HOST_DEVICE_API auto __receiver_proxy_base<_Rcvr, _Proxy>::query(const get_scheduler_t&) const noexcept
|
||||
-> task_scheduler
|
||||
{
|
||||
if constexpr (__callable<const get_scheduler_t&, env_of_t<_Rcvr>>)
|
||||
{
|
||||
return task_scheduler{get_scheduler(get_env(__rcvr_))};
|
||||
}
|
||||
else
|
||||
{
|
||||
return task_scheduler{inline_scheduler{}};
|
||||
}
|
||||
}
|
||||
} // namespace __detail
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_TASK_SCHEDULER
|
||||
@@ -1,409 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_THEN
|
||||
#define __CUDAX_EXECUTION_THEN
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__cccl/unreachable.h>
|
||||
#include <cuda/std/__exception/exception_macros.h>
|
||||
#include <cuda/std/__type_traits/is_same.h>
|
||||
#include <cuda/std/__type_traits/type_list.h>
|
||||
#include <cuda/std/__utility/pod_tuple.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/concepts.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/exception.cuh>
|
||||
#include <cuda/experimental/__execution/meta.cuh>
|
||||
#include <cuda/experimental/__execution/rcvr_ref.cuh>
|
||||
#include <cuda/experimental/__execution/transform_sender.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(4702) // warning C4702: unreachable code
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
namespace __upon
|
||||
{
|
||||
template <bool IsVoid, bool _Nothrow>
|
||||
struct __completion_fn
|
||||
{ // non-void, potentially throwing case
|
||||
template <class _Result>
|
||||
using __call _CCCL_NODEBUG_ALIAS = completion_signatures<set_value_t(_Result), set_error_t(exception_ptr)>;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct __completion_fn<true, false>
|
||||
{ // void, potentially throwing case
|
||||
template <class>
|
||||
using __call _CCCL_NODEBUG_ALIAS = completion_signatures<set_value_t(), set_error_t(exception_ptr)>;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct __completion_fn<false, true>
|
||||
{ // non-void, non-throwing case
|
||||
template <class _Result>
|
||||
using __call _CCCL_NODEBUG_ALIAS = completion_signatures<set_value_t(_Result)>;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct __completion_fn<true, true>
|
||||
{ // void, non-throwing case
|
||||
template <class>
|
||||
using __call _CCCL_NODEBUG_ALIAS = completion_signatures<set_value_t()>;
|
||||
};
|
||||
|
||||
template <class _Result, bool _Nothrow>
|
||||
using __completion_ _CCCL_NODEBUG_ALIAS =
|
||||
::cuda::std::__type_call1<__completion_fn<__same_as<_Result, void>, _Nothrow>, _Result>;
|
||||
|
||||
template <class _Fn, class... _Ts>
|
||||
using __completion _CCCL_NODEBUG_ALIAS = __completion_<__call_result_t<_Fn, _Ts...>, __nothrow_callable<_Fn, _Ts...>>;
|
||||
|
||||
template <class _Fn, class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __state_t
|
||||
{
|
||||
_Rcvr __rcvr_;
|
||||
_Fn __fn_;
|
||||
};
|
||||
} // namespace __upon
|
||||
|
||||
template <class _UponTag, class _SetTag>
|
||||
struct __upon_t
|
||||
{
|
||||
_CUDAX_SEMI_PRIVATE :
|
||||
friend struct then_t;
|
||||
friend struct upon_error_t;
|
||||
friend struct upon_stopped_t;
|
||||
|
||||
using __upon_tag_t = _UponTag;
|
||||
using __set_tag_t = _SetTag;
|
||||
|
||||
template <class _Fn, class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_t
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API void __set(_Ts&&... __ts) noexcept
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
if constexpr (__same_as<void, __call_result_t<_Fn, _Ts...>>)
|
||||
{
|
||||
static_cast<_Fn&&>(__state_->__fn_)(static_cast<_Ts&&>(__ts)...);
|
||||
execution::set_value(static_cast<_Rcvr&&>(__state_->__rcvr_));
|
||||
}
|
||||
else
|
||||
{
|
||||
// msvc warns that this is unreachable code, but it is reachable.
|
||||
execution::set_value(static_cast<_Rcvr&&>(__state_->__rcvr_),
|
||||
static_cast<_Fn&&>(__state_->__fn_)(static_cast<_Ts&&>(__ts)...));
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_ALL
|
||||
{
|
||||
if constexpr (!__nothrow_callable<_Fn, _Ts...>)
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(__state_->__rcvr_), execution::current_exception());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Tag, class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API void __complete(_Tag, _Ts&&... __ts) noexcept
|
||||
{
|
||||
if constexpr (_Tag{} == _SetTag{})
|
||||
{
|
||||
__set(static_cast<_Ts&&>(__ts)...);
|
||||
}
|
||||
else
|
||||
{
|
||||
_Tag{}(static_cast<_Rcvr&&>(__state_->__rcvr_), static_cast<_Ts&&>(__ts)...);
|
||||
}
|
||||
}
|
||||
|
||||
template <class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API void set_value(_Ts&&... __ts) noexcept
|
||||
{
|
||||
__complete(set_value_t{}, static_cast<_Ts&&>(__ts)...);
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API void set_error(_Error&& __error) noexcept
|
||||
{
|
||||
__complete(set_error_t{}, static_cast<_Error&&>(__error));
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void set_stopped() noexcept
|
||||
{
|
||||
__complete(set_stopped_t{});
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __fwd_env_t<env_of_t<_Rcvr>>
|
||||
{
|
||||
return __fwd_env(execution::get_env(__state_->__rcvr_));
|
||||
}
|
||||
|
||||
__upon::__state_t<_Fn, _Rcvr>* __state_;
|
||||
};
|
||||
|
||||
template <class _CvSndr, class _Fn, class _Rcvr>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
using __rcvr_t = __upon_t::__rcvr_t<_Fn, _Rcvr>;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __opstate_t(_CvSndr&& __sndr, _Rcvr __rcvr, _Fn __fn)
|
||||
: __state_{static_cast<_Rcvr&&>(__rcvr), static_cast<_Fn&&>(__fn)}
|
||||
, __opstate_{execution::connect(static_cast<_CvSndr&&>(__sndr), __rcvr_t{&__state_})}
|
||||
{}
|
||||
|
||||
_CCCL_IMMOVABLE(__opstate_t);
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void start() noexcept
|
||||
{
|
||||
execution::start(__opstate_);
|
||||
}
|
||||
|
||||
__upon::__state_t<_Fn, _Rcvr> __state_;
|
||||
connect_result_t<_CvSndr, __rcvr_t> __opstate_;
|
||||
};
|
||||
|
||||
template <class _Fn>
|
||||
struct __transform_args_fn
|
||||
{
|
||||
template <class... _Ts>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto operator()() const
|
||||
{
|
||||
if constexpr (__callable<_Fn, _Ts...>)
|
||||
{
|
||||
return __upon::__completion<_Fn, _Ts...>{};
|
||||
}
|
||||
else
|
||||
{
|
||||
return invalid_completion_signature<_WHERE(_IN_ALGORITHM, _UponTag),
|
||||
_WHAT(_FUNCTION_IS_NOT_CALLABLE),
|
||||
_WITH_FUNCTION(_Fn),
|
||||
_WITH_ARGUMENTS(_Ts...)>();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_base_t;
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_HIDDEN __closure_base_t // hidden visibility because member __fn_ is hidden if it is an
|
||||
// extended (host/device) lambda
|
||||
{
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr __sndr) && -> __call_result_t<__upon_tag_t, _Sndr, _Fn>
|
||||
{
|
||||
return __upon_tag_t{}(static_cast<_Sndr&&>(__sndr), static_cast<_Fn&&>(__fn_));
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr __sndr) const& -> __call_result_t<__upon_tag_t, _Sndr, _Fn>
|
||||
{
|
||||
return __upon_tag_t{}(static_cast<_Sndr&&>(__sndr), __fn_);
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
_CCCL_HOST_DEVICE_API friend constexpr auto operator|(_Sndr __sndr, __closure_base_t __self) //
|
||||
-> __call_result_t<__upon_tag_t, _Sndr, _Fn>
|
||||
{
|
||||
return __upon_tag_t{}(static_cast<_Sndr&&>(__sndr), static_cast<_Fn&&>(__self.__fn_));
|
||||
}
|
||||
|
||||
_Fn __fn_;
|
||||
};
|
||||
|
||||
public:
|
||||
template <class _Sndr, class _Fn>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr __sndr, _Fn __fn) const;
|
||||
|
||||
template <class _Fn>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Fn __fn) const;
|
||||
};
|
||||
|
||||
struct then_t : __upon_t<then_t, set_value_t>
|
||||
{
|
||||
template <class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t;
|
||||
};
|
||||
|
||||
struct upon_error_t : __upon_t<upon_error_t, set_error_t>
|
||||
{
|
||||
template <class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t;
|
||||
};
|
||||
|
||||
struct upon_stopped_t : __upon_t<upon_stopped_t, set_stopped_t>
|
||||
{
|
||||
template <class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t;
|
||||
};
|
||||
|
||||
template <class _UponTag, class _SetTag>
|
||||
template <class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __upon_t<_UponTag, _SetTag>::__sndr_base_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class _Self, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
_CUDAX_LET_COMPLETIONS(auto(__child_completions) = get_child_completion_signatures<_Self, _Sndr, _Env...>())
|
||||
{
|
||||
if constexpr (__set_tag_t{} == execution::set_value)
|
||||
{
|
||||
return transform_completion_signatures(__child_completions, __transform_args_fn<_Fn>{});
|
||||
}
|
||||
else if constexpr (__set_tag_t{} == execution::set_error)
|
||||
{
|
||||
return transform_completion_signatures(__child_completions, {}, __transform_args_fn<_Fn>{});
|
||||
}
|
||||
else
|
||||
{
|
||||
return transform_completion_signatures(__child_completions, {}, {}, __transform_args_fn<_Fn>{});
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) && //
|
||||
noexcept(__nothrow_constructible<__opstate_t<_Sndr, _Fn, _Rcvr>, _Sndr, _Rcvr, _Fn>) //
|
||||
-> __opstate_t<_Sndr, _Fn, _Rcvr>
|
||||
{
|
||||
return __opstate_t<_Sndr, _Fn, _Rcvr>{
|
||||
static_cast<_Sndr&&>(__sndr_), static_cast<_Rcvr&&>(__rcvr), static_cast<_Fn&&>(__fn_)};
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) const& //
|
||||
noexcept(__nothrow_constructible<__opstate_t<_Sndr const&, _Fn, _Rcvr>,
|
||||
const _Sndr&,
|
||||
_Rcvr,
|
||||
const _Fn&>) //
|
||||
-> __opstate_t<_Sndr const&, _Fn, _Rcvr>
|
||||
{
|
||||
return __opstate_t<_Sndr const&, _Fn, _Rcvr>{__sndr_, static_cast<_Rcvr&&>(__rcvr), __fn_};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __fwd_env_t<env_of_t<_Sndr>>
|
||||
{
|
||||
return __fwd_env(execution::get_env(__sndr_));
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ __upon_tag_t __tag_;
|
||||
_Fn __fn_;
|
||||
_Sndr __sndr_;
|
||||
};
|
||||
|
||||
template <class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT then_t::__sndr_t : __upon_t<then_t, set_value_t>::__sndr_base_t<_Sndr, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT then_t::__closure_t : __upon_t<then_t, set_value_t>::__closure_base_t<_Fn>
|
||||
{};
|
||||
|
||||
template <class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT upon_error_t::__sndr_t
|
||||
: __upon_t<upon_error_t, set_error_t>::__sndr_base_t<_Sndr, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT upon_error_t::__closure_t
|
||||
: __upon_t<upon_error_t, set_error_t>::__closure_base_t<_Fn>
|
||||
{};
|
||||
|
||||
template <class _Sndr, class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT upon_stopped_t::__sndr_t
|
||||
: __upon_t<upon_stopped_t, set_stopped_t>::__sndr_base_t<_Sndr, _Fn>
|
||||
{};
|
||||
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT upon_stopped_t::__closure_t
|
||||
: __upon_t<upon_stopped_t, set_stopped_t>::__closure_base_t<_Fn>
|
||||
{};
|
||||
|
||||
template <class _UponTag, class _SetTag>
|
||||
template <class _Sndr, class _Fn>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto __upon_t<_UponTag, _SetTag>::operator()(_Sndr __sndr, _Fn __fn) const
|
||||
{
|
||||
using __sndr_t = typename _UponTag::template __sndr_t<_Sndr, _Fn>;
|
||||
|
||||
// If the incoming sender is non-dependent, we can check the completion
|
||||
// signatures of the composed sender immediately.
|
||||
if constexpr (!dependent_sender<_Sndr>)
|
||||
{
|
||||
__assert_valid_completion_signatures(get_completion_signatures<__sndr_t>());
|
||||
}
|
||||
|
||||
return __sndr_t{{{}, static_cast<_Fn&&>(__fn), static_cast<_Sndr&&>(__sndr)}};
|
||||
}
|
||||
|
||||
template <class _UponTag, class _SetTag>
|
||||
template <class _Fn>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto __upon_t<_UponTag, _SetTag>::operator()(_Fn __fn) const
|
||||
{
|
||||
using __closure_t = typename _UponTag::template __closure_t<_Fn>;
|
||||
return __closure_t{{static_cast<_Fn&&>(__fn)}};
|
||||
}
|
||||
|
||||
template <class _Sndr, class _Fn>
|
||||
inline constexpr int structured_binding_size<then_t::__sndr_t<_Sndr, _Fn>> = 3;
|
||||
template <class _Sndr, class _Fn>
|
||||
inline constexpr int structured_binding_size<upon_error_t::__sndr_t<_Sndr, _Fn>> = 3;
|
||||
template <class _Sndr, class _Fn>
|
||||
inline constexpr int structured_binding_size<upon_stopped_t::__sndr_t<_Sndr, _Fn>> = 3;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto then = then_t{};
|
||||
_CCCL_GLOBAL_CONSTANT auto upon_error = upon_error_t{};
|
||||
_CCCL_GLOBAL_CONSTANT auto upon_stopped = upon_stopped_t{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_THEN
|
||||
@@ -1,87 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_THREAD
|
||||
#define __CUDAX_EXECUTION_THREAD
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <thread>
|
||||
|
||||
#if _CCCL_CUDA_COMPILATION()
|
||||
# include <nv/target>
|
||||
# define _CUDAX_FOR_HOST_OR_DEVICE(_FOR_HOST, _FOR_DEVICE) NV_IF_ELSE_TARGET(NV_IS_HOST, _FOR_HOST, _FOR_DEVICE)
|
||||
#else // ^^^ _CCCL_CUDA_COMPILATION() ^^^ / vvv !_CCCL_CUDA_COMPILATION() vvv
|
||||
# define _CUDAX_FOR_HOST_OR_DEVICE(_FOR_HOST, _FOR_DEVICE) {_CCCL_PP_EXPAND _FOR_HOST}
|
||||
#endif // ^^^ !_CCCL_CUDA_COMPILATION() ^^^
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
#if _CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC)
|
||||
using __thread_id _CCCL_NODEBUG_ALIAS = int;
|
||||
#elif _CCCL_CUDA_COMPILER(NVHPC)
|
||||
struct __thread_id
|
||||
{
|
||||
union
|
||||
{
|
||||
::std::thread::id __host_;
|
||||
int __device_;
|
||||
};
|
||||
|
||||
_CCCL_HOST_DEVICE_API __thread_id() noexcept
|
||||
: __host_()
|
||||
{}
|
||||
_CCCL_HOST_DEVICE_API __thread_id(::std::thread::id __host) noexcept
|
||||
: __host_(__host)
|
||||
{}
|
||||
_CCCL_HOST_DEVICE_API __thread_id(int __device) noexcept
|
||||
: __device_(__device)
|
||||
{}
|
||||
|
||||
_CCCL_HOST_DEVICE_API friend bool operator==(const __thread_id& __self, const __thread_id& __other) noexcept
|
||||
{
|
||||
_CUDAX_FOR_HOST_OR_DEVICE((return __self.__host_ == __other.__host_;),
|
||||
(return __self.__device_ == __other.__device_;))
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API friend bool operator!=(const __thread_id& __self, const __thread_id& __other) noexcept
|
||||
{
|
||||
return !(__self == __other);
|
||||
}
|
||||
};
|
||||
#else // ^^^ cuda device compilation ^^^ / vvv host compilation vvv
|
||||
using __thread_id _CCCL_NODEBUG_ALIAS = ::std::thread::id;
|
||||
#endif // ^^^ host compilation ^^^
|
||||
|
||||
inline _CCCL_HOST_DEVICE_API auto __this_thread_id() noexcept -> __thread_id
|
||||
{
|
||||
_CUDAX_FOR_HOST_OR_DEVICE((return ::std::this_thread::get_id();),
|
||||
(return static_cast<int>(threadIdx.x + blockIdx.x * blockDim.x);))
|
||||
}
|
||||
|
||||
inline _CCCL_HOST_DEVICE_API void __this_thread_yield() noexcept
|
||||
{
|
||||
_CUDAX_FOR_HOST_OR_DEVICE((::std::this_thread::yield();), (void();))
|
||||
}
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_THREAD
|
||||
@@ -1,72 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_THREAD_CONTEXT
|
||||
#define __CUDAX_EXECUTION_THREAD_CONTEXT
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/experimental/__execution/run_loop.cuh>
|
||||
|
||||
#include <thread>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT thread_context
|
||||
{
|
||||
_CCCL_HOST_API thread_context() noexcept
|
||||
: __thrd_{[this] {
|
||||
__loop_.run();
|
||||
}}
|
||||
{}
|
||||
|
||||
_CCCL_HOST_API ~thread_context() noexcept
|
||||
{
|
||||
join();
|
||||
}
|
||||
|
||||
_CCCL_HOST_API void join() noexcept
|
||||
{
|
||||
if (__thrd_.joinable())
|
||||
{
|
||||
__loop_.finish();
|
||||
__thrd_.join();
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API auto get_scheduler()
|
||||
{
|
||||
return __loop_.get_scheduler();
|
||||
}
|
||||
|
||||
_CCCL_HOST_API auto get_id() const noexcept
|
||||
{
|
||||
return __thrd_.get_id();
|
||||
}
|
||||
|
||||
private:
|
||||
run_loop __loop_;
|
||||
::std::thread __thrd_;
|
||||
};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_THREAD_CONTEXT
|
||||
@@ -1,322 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Facebook, Inc. and its affiliates.
|
||||
* Copyright (c) 2025 NVIDIA Corporation
|
||||
*
|
||||
* Licensed under the Apache License Version 2.0 with LLVM Exceptions
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* https://llvm.org/LICENSE.txt
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_TRAMPOLINE_SCHEDULER
|
||||
#define __CUDAX_EXECUTION_TRAMPOLINE_SCHEDULER
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__cstdlib/abs.h>
|
||||
#include <cuda/std/__tuple_dir/ignore.h>
|
||||
#include <cuda/std/__type_traits/conditional.h>
|
||||
#include <cuda/std/__utility/exchange.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/concepts.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/stop_token.cuh>
|
||||
|
||||
// include this last:
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
namespace __detail
|
||||
{
|
||||
template <class _Operation>
|
||||
struct __trampoline_state
|
||||
{
|
||||
static thread_local __trampoline_state* __current_;
|
||||
|
||||
_CCCL_HOST_API __trampoline_state(size_t __max_recursion_depth, size_t __max_recursion_size) noexcept
|
||||
: __max_recursion_size_(__max_recursion_size)
|
||||
, __max_recursion_depth_(__max_recursion_depth)
|
||||
{
|
||||
__current_ = this;
|
||||
}
|
||||
|
||||
_CCCL_HOST_API ~__trampoline_state()
|
||||
{
|
||||
__current_ = nullptr;
|
||||
}
|
||||
|
||||
_CCCL_HOST_API void __drain() noexcept;
|
||||
|
||||
// these origin schedule frame limits will apply to all
|
||||
// nested trampoline instances on this thread
|
||||
const size_t __max_recursion_size_;
|
||||
const size_t __max_recursion_depth_;
|
||||
|
||||
// track state of origin schedule frame
|
||||
intptr_t __recursion_origin_ = 0;
|
||||
size_t __recursion_depth_ = 1;
|
||||
_Operation* __head_ = nullptr;
|
||||
_Operation* __tail_ = nullptr;
|
||||
};
|
||||
} // namespace __detail
|
||||
|
||||
class trampoline_scheduler
|
||||
{
|
||||
struct __schedule_sender;
|
||||
|
||||
public:
|
||||
using scheduler_concept = scheduler_t;
|
||||
|
||||
_CCCL_HOST_API trampoline_scheduler() noexcept
|
||||
: __max_recursion_size_(4096)
|
||||
, __max_recursion_depth_(16)
|
||||
{}
|
||||
|
||||
_CCCL_HOST_API explicit trampoline_scheduler(size_t __max_recursion_depth) noexcept
|
||||
: __max_recursion_size_(4096)
|
||||
, __max_recursion_depth_(__max_recursion_depth)
|
||||
{}
|
||||
|
||||
_CCCL_HOST_API explicit trampoline_scheduler(size_t __max_recursion_depth, size_t __max_recursion_size) noexcept
|
||||
: __max_recursion_size_(__max_recursion_size)
|
||||
, __max_recursion_depth_(__max_recursion_depth)
|
||||
{}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_API auto schedule() const noexcept -> __schedule_sender
|
||||
{
|
||||
return __schedule_sender{__max_recursion_size_, __max_recursion_depth_};
|
||||
}
|
||||
|
||||
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
|
||||
|
||||
_CCCL_HOST_API auto operator==(const trampoline_scheduler&) const noexcept -> bool = default;
|
||||
|
||||
#else
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_API friend constexpr bool
|
||||
operator==(const trampoline_scheduler& __a, const trampoline_scheduler& __b) noexcept
|
||||
{
|
||||
return __a.__max_recursion_size_ == __b.__max_recursion_size_
|
||||
&& __a.__max_recursion_depth_ == __b.__max_recursion_depth_;
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_API friend constexpr bool
|
||||
operator!=(const trampoline_scheduler& __a, const trampoline_scheduler& __b) noexcept
|
||||
{
|
||||
return !(__a == __b);
|
||||
}
|
||||
|
||||
#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
|
||||
|
||||
private:
|
||||
struct __operation_base
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
using __execute_fn = void(__operation_base*) noexcept;
|
||||
|
||||
_CCCL_HOST_API explicit __operation_base(__execute_fn* __execute, size_t __max_size, size_t __max_depth) noexcept
|
||||
: __execute_(__execute)
|
||||
, __max_recursion_size_(__max_size)
|
||||
, __max_recursion_depth_(__max_depth)
|
||||
{}
|
||||
|
||||
_CCCL_HOST_API void __execute() noexcept
|
||||
{
|
||||
__execute_(this);
|
||||
}
|
||||
|
||||
_CCCL_HOST_API void start() & noexcept
|
||||
{
|
||||
auto* __current_state = __detail::__trampoline_state<__operation_base>::__current_;
|
||||
|
||||
if (__current_state == nullptr)
|
||||
{
|
||||
// origin schedule frame on this thread
|
||||
__detail::__trampoline_state<__operation_base> __state{__max_recursion_depth_, __max_recursion_size_};
|
||||
__execute();
|
||||
__state.__drain();
|
||||
}
|
||||
else
|
||||
{
|
||||
// recursive schedule frame on this thread
|
||||
|
||||
// calculate stack consumption for this schedule
|
||||
size_t __current_size =
|
||||
::cuda::std::abs(reinterpret_cast<intptr_t>(&__current_state) - __current_state->__recursion_origin_);
|
||||
|
||||
if (__current_size < __current_state->__max_recursion_size_
|
||||
&& __current_state->__recursion_depth_ < __current_state->__max_recursion_depth_)
|
||||
{
|
||||
// inline this recursive schedule
|
||||
++__current_state->__recursion_depth_;
|
||||
__execute();
|
||||
}
|
||||
else
|
||||
{
|
||||
// Exceeded recursion limit.
|
||||
|
||||
// push this recursive schedule to list tail
|
||||
__prev_ = ::cuda::std::exchange(__current_state->__tail_, static_cast<__operation_base*>(this));
|
||||
if (__prev_ != nullptr)
|
||||
{
|
||||
// was not empty
|
||||
::cuda::std::exchange(__prev_->__next_, static_cast<__operation_base*>(this));
|
||||
}
|
||||
else
|
||||
{
|
||||
// was empty
|
||||
::cuda::std::exchange(__current_state->__head_, static_cast<__operation_base*>(this));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
__operation_base* __prev_ = nullptr;
|
||||
__operation_base* __next_ = nullptr;
|
||||
__execute_fn* __execute_;
|
||||
const size_t __max_recursion_size_;
|
||||
const size_t __max_recursion_depth_;
|
||||
};
|
||||
|
||||
template <class _Rcvr>
|
||||
struct __operation : __operation_base
|
||||
{
|
||||
_CCCL_HOST_API explicit __operation(_Rcvr __rcvr, size_t __max_size, size_t __max_depth) noexcept
|
||||
: __operation_base(&__operation::__execute_impl, __max_size, __max_depth)
|
||||
, __rcvr_(static_cast<_Rcvr&&>(__rcvr))
|
||||
{}
|
||||
|
||||
_CCCL_HOST_API static void __execute_impl(__operation_base* __op) noexcept
|
||||
{
|
||||
auto& __self = *static_cast<__operation*>(__op);
|
||||
if constexpr (unstoppable_token<stop_token_of_t<env_of_t<_Rcvr&>>>)
|
||||
{
|
||||
execution::set_value(static_cast<_Rcvr&&>(__self.__rcvr_));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (execution::get_stop_token(get_env(__self.__rcvr_)).stop_requested())
|
||||
{
|
||||
execution::set_stopped(static_cast<_Rcvr&&>(__self.__rcvr_));
|
||||
}
|
||||
else
|
||||
{
|
||||
execution::set_value(static_cast<_Rcvr&&>(__self.__rcvr_));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
_Rcvr __rcvr_;
|
||||
};
|
||||
|
||||
struct __schedule_sender
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
template <class _Env>
|
||||
using __completions_in_t =
|
||||
::cuda::std::_If<unstoppable_token<stop_token_of_t<_Env>>,
|
||||
completion_signatures<set_value_t()>,
|
||||
completion_signatures<set_value_t(), set_stopped_t()>>;
|
||||
|
||||
_CCCL_HOST_API explicit __schedule_sender(size_t __max_size, size_t __max_depth) noexcept
|
||||
: __max_recursion_size_(__max_size)
|
||||
, __max_recursion_depth_(__max_depth)
|
||||
{}
|
||||
|
||||
_CCCL_TEMPLATE(class _Rcvr)
|
||||
_CCCL_REQUIRES(receiver_of<_Rcvr, __completions_in_t<env_of_t<_Rcvr&>>>)
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_API auto connect(_Rcvr __rcvr) const noexcept -> __operation<_Rcvr>
|
||||
{
|
||||
return __operation<_Rcvr>{static_cast<_Rcvr&&>(__rcvr), __max_recursion_size_, __max_recursion_depth_};
|
||||
}
|
||||
|
||||
template <class _Self, class _Env>
|
||||
_CCCL_HOST_API static _CCCL_CONSTEVAL auto get_completion_signatures() noexcept -> __completions_in_t<_Env>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_API auto query(get_completion_scheduler_t<set_value_t>, ::cuda::std::__ignore_t = {}) const noexcept
|
||||
-> trampoline_scheduler
|
||||
{
|
||||
return trampoline_scheduler{__max_recursion_depth_};
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_HOST_API auto get_env() const noexcept -> const __schedule_sender&
|
||||
{
|
||||
return *this;
|
||||
}
|
||||
|
||||
const size_t __max_recursion_size_;
|
||||
const size_t __max_recursion_depth_;
|
||||
};
|
||||
|
||||
size_t __max_recursion_size_;
|
||||
size_t __max_recursion_depth_;
|
||||
};
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
template <class _Operation>
|
||||
thread_local __trampoline_state<_Operation>* __trampoline_state<_Operation>::__current_ = nullptr;
|
||||
|
||||
template <class _Operation>
|
||||
_CCCL_HOST_API void __trampoline_state<_Operation>::__drain() noexcept
|
||||
{
|
||||
while (__head_ != nullptr)
|
||||
{
|
||||
// pop the head of the list
|
||||
_Operation* __op = ::cuda::std::exchange(__head_, __head_->__next_);
|
||||
__op->__next_ = nullptr;
|
||||
__op->__prev_ = nullptr;
|
||||
if (__head_ != nullptr)
|
||||
{
|
||||
// is not empty
|
||||
__head_->__prev_ = nullptr;
|
||||
}
|
||||
else
|
||||
{
|
||||
// is empty
|
||||
__tail_ = nullptr;
|
||||
}
|
||||
|
||||
// reset the origin schedule frame state
|
||||
__recursion_origin_ = reinterpret_cast<intptr_t>(&__op);
|
||||
__recursion_depth_ = 1;
|
||||
|
||||
__op->__execute();
|
||||
}
|
||||
}
|
||||
} // namespace __detail
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_TRAMPOLINE_SCHEDULER
|
||||
@@ -1,196 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_TRANSFORM_COMPLETION_SIGNATURES
|
||||
#define __CUDAX_EXECUTION_TRANSFORM_COMPLETION_SIGNATURES
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__type_traits/decay.h>
|
||||
#include <cuda/std/__type_traits/is_base_of.h>
|
||||
#include <cuda/std/__type_traits/type_list.h>
|
||||
#include <cuda/std/__type_traits/type_set.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh> // IWYU pragma: export
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/get_completion_signatures.cuh>
|
||||
|
||||
// include this last:
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
template <class _Tag>
|
||||
struct __default_transform_fn
|
||||
{
|
||||
template <class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto operator()() const noexcept -> completion_signatures<_Tag(_Ts...)>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
struct __swallow_transform
|
||||
{
|
||||
template <class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto operator()() const noexcept -> completion_signatures<>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Tag>
|
||||
struct __decay_transform
|
||||
{
|
||||
template <class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto operator()() const noexcept -> completion_signatures<_Tag(decay_t<_Ts>...)>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(4864) // nvbug5765092 latest toolchain complains about missing template
|
||||
|
||||
template <class _Fn, class... _As>
|
||||
using __meta_call_result_t _CCCL_NODEBUG_ALIAS = decltype(declval<_Fn>().template operator()<_As...>());
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Ay, class... _As, class _Fn>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __transform_expr(const _Fn& __fn)
|
||||
-> __meta_call_result_t<const _Fn&, _Ay, _As...>
|
||||
{
|
||||
return __fn.template operator()<_Ay, _As...>();
|
||||
}
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Fn>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __transform_expr(const _Fn& __fn)
|
||||
-> __call_result_t<const _Fn&>
|
||||
{
|
||||
return __fn();
|
||||
}
|
||||
|
||||
template <class _Fn, class... _As>
|
||||
using __transform_expr_t _CCCL_NODEBUG_ALIAS = decltype(execution::__transform_expr<_As...>(declval<const _Fn&>()));
|
||||
|
||||
struct _IN_TRANSFORM_COMPLETION_SIGNATURES;
|
||||
struct _A_TRANSFORM_FUNCTION_RETURNED_A_TYPE_THAT_IS_NOT_A_COMPLETION_SIGNATURES_SPECIALIZATION;
|
||||
struct _COULD_NOT_CALL_THE_TRANSFORM_FUNCTION_WITH_THE_GIVEN_TEMPLATE_ARGUMENTS;
|
||||
|
||||
// transform_completion_signatures:
|
||||
template <class... _As, class _Fn>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __apply_transform(const _Fn& __fn)
|
||||
{
|
||||
if constexpr (__is_instantiable_with<__transform_expr_t, _Fn, _As...>)
|
||||
{
|
||||
using __completions _CCCL_NODEBUG_ALIAS = __transform_expr_t<_Fn, _As...>;
|
||||
if constexpr (__valid_completion_signatures<__completions> || __type_is_error<__completions>
|
||||
|| ::cuda::std::is_base_of_v<dependent_sender_error, __completions>)
|
||||
{
|
||||
return execution::__transform_expr<_As...>(__fn);
|
||||
}
|
||||
else
|
||||
{
|
||||
(void) execution::__transform_expr<_As...>(__fn); // potentially throwing
|
||||
return invalid_completion_signature<
|
||||
_IN_TRANSFORM_COMPLETION_SIGNATURES,
|
||||
_A_TRANSFORM_FUNCTION_RETURNED_A_TYPE_THAT_IS_NOT_A_COMPLETION_SIGNATURES_SPECIALIZATION,
|
||||
_WITH_FUNCTION(_Fn),
|
||||
_WITH_ARGUMENTS(_As...)>();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return invalid_completion_signature< //
|
||||
_IN_TRANSFORM_COMPLETION_SIGNATURES,
|
||||
_COULD_NOT_CALL_THE_TRANSFORM_FUNCTION_WITH_THE_GIVEN_TEMPLATE_ARGUMENTS,
|
||||
_WITH_FUNCTION(_Fn),
|
||||
_WITH_ARGUMENTS(_As...)>();
|
||||
}
|
||||
}
|
||||
|
||||
template <class _ValueFn, class _ErrorFn, class _StoppedFn>
|
||||
struct __transform_one
|
||||
{
|
||||
_ValueFn __value_fn;
|
||||
_ErrorFn __error_fn;
|
||||
_StoppedFn __stopped_fn;
|
||||
|
||||
template <class _Tag, class... _Ts>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto operator()(_Tag (*)(_Ts...)) const
|
||||
{
|
||||
if constexpr (_Tag{} == set_value)
|
||||
{
|
||||
return __apply_transform<_Ts...>(__value_fn);
|
||||
}
|
||||
else if constexpr (_Tag{} == set_error)
|
||||
{
|
||||
return __apply_transform<_Ts...>(__error_fn);
|
||||
}
|
||||
else
|
||||
{
|
||||
return __apply_transform<_Ts...>(__stopped_fn);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class _TransformOne>
|
||||
struct __transform_all_fn
|
||||
{
|
||||
_TransformOne __tfx1;
|
||||
|
||||
template <class... _Sigs>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto operator()(_Sigs*... __sigs) const
|
||||
{
|
||||
return concat_completion_signatures(__tfx1(__sigs)...);
|
||||
}
|
||||
};
|
||||
|
||||
template <class _TransformOne>
|
||||
__transform_all_fn(_TransformOne) -> __transform_all_fn<_TransformOne>;
|
||||
|
||||
template <class _Completions,
|
||||
class _ValueFn = __default_transform_fn<set_value_t>,
|
||||
class _ErrorFn = __default_transform_fn<set_error_t>,
|
||||
class _StoppedFn = __default_transform_fn<set_stopped_t>,
|
||||
class _ExtraSigs = completion_signatures<>>
|
||||
_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto transform_completion_signatures(
|
||||
_Completions, //
|
||||
_ValueFn __value_fn = {},
|
||||
_ErrorFn __error_fn = {},
|
||||
_StoppedFn __stopped_fn = {},
|
||||
_ExtraSigs = {})
|
||||
{
|
||||
_CUDAX_LET_COMPLETIONS(auto(__completions) = _Completions{})
|
||||
{
|
||||
_CUDAX_LET_COMPLETIONS(auto(__extra) = _ExtraSigs{})
|
||||
{
|
||||
__transform_one<_ValueFn, _ErrorFn, _StoppedFn> __tfx1{__value_fn, __error_fn, __stopped_fn};
|
||||
return concat_completion_signatures(__completions.apply(__transform_all_fn{__tfx1}), __extra);
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_TRANSFORM_COMPLETION_SIGNATURES
|
||||
@@ -1,138 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_TRANSFORM_SENDER
|
||||
#define __CUDAX_EXECUTION_TRANSFORM_SENDER
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__type_traits/conditional.h>
|
||||
#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
|
||||
#include <cuda/std/__type_traits/is_valid_expansion.h>
|
||||
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/domain.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/fwd.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
namespace __detail
|
||||
{
|
||||
template <class _Env>
|
||||
using __starting_domain = __domain_of_t<const _Env&>;
|
||||
|
||||
template <class _Sndr, class _Env>
|
||||
using __completing_domain = __call_result_t<get_completion_domain_t<set_value_t>, env_of_t<_Sndr>, const _Env&>;
|
||||
|
||||
template <class _Domain, class _OpTag>
|
||||
struct __transform_sender_t
|
||||
{
|
||||
template <class _Sndr, class _Env>
|
||||
using __domain_for_t = ::cuda::std::_If< //
|
||||
__has_transform_sender<_Domain, _OpTag, _Sndr, _Env>,
|
||||
_Domain,
|
||||
default_domain>;
|
||||
|
||||
template <class _Sndr, class _Env, bool _Nothrow = true>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto __get_declfn() noexcept
|
||||
{
|
||||
using __domain_t = __domain_for_t<_Sndr, _Env>;
|
||||
using __result_t = __transform_sender_result_t<__domain_t, _OpTag, _Sndr, _Env>;
|
||||
|
||||
constexpr bool __is_nothrow = __nothrow_transform_sender<__domain_t, _OpTag, _Sndr, _Env>;
|
||||
|
||||
if constexpr (__same_as<__result_t, _Sndr>)
|
||||
{
|
||||
return __declfn<__result_t, __is_nothrow>;
|
||||
}
|
||||
else if constexpr (__same_as<_OpTag, start_t>)
|
||||
{
|
||||
return __get_declfn<__result_t, const _Env&, (_Nothrow && __is_nothrow)>();
|
||||
}
|
||||
else
|
||||
{
|
||||
using __transform_recurse_t = __transform_sender_t<__completing_domain<__result_t, _Env>, set_value_t>;
|
||||
return __transform_recurse_t::template __get_declfn<__result_t, _Env, (_Nothrow && __is_nothrow)>();
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Sndr, class _Env, auto _DeclFn = __get_declfn<_Sndr, _Env>()>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr&& __sndr, const _Env& __env) const
|
||||
noexcept(noexcept(_DeclFn())) -> decltype(_DeclFn())
|
||||
{
|
||||
using __domain_t = __domain_for_t<_Sndr, _Env>;
|
||||
using __result_t = __transform_sender_result_t<__domain_t, _OpTag, _Sndr, _Env>;
|
||||
|
||||
if constexpr (__same_as<__result_t, _Sndr>)
|
||||
{
|
||||
return __domain_t().transform_sender(_OpTag(), static_cast<_Sndr&&>(__sndr), __env);
|
||||
}
|
||||
else if constexpr (__same_as<_OpTag, start_t>)
|
||||
{
|
||||
return (*this)(__domain_t().transform_sender(_OpTag(), static_cast<_Sndr&&>(__sndr), __env), __env);
|
||||
}
|
||||
else
|
||||
{
|
||||
using __transform_recurse_t = __transform_sender_t<__completing_domain<__result_t, _Env>, set_value_t>;
|
||||
return __transform_recurse_t()(
|
||||
__domain_t().transform_sender(_OpTag(), static_cast<_Sndr&&>(__sndr), __env), __env);
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace __detail
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT transform_sender_t
|
||||
{
|
||||
private:
|
||||
template <class _Fn1, class _Fn2>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __compose
|
||||
{
|
||||
template <class _Sndr, class _Env>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr&& __sndr, const _Env& __env) const
|
||||
noexcept(noexcept(_Fn1()(_Fn2()(static_cast<_Sndr&&>(__sndr), __env), __env)))
|
||||
-> decltype(_Fn1()(_Fn2()(static_cast<_Sndr&&>(__sndr), __env), __env))
|
||||
{
|
||||
return _Fn1()(_Fn2()(static_cast<_Sndr&&>(__sndr), __env), __env);
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Sndr, class _Env>
|
||||
using __impl_fn_t =
|
||||
__compose<__detail::__transform_sender_t<__detail::__starting_domain<_Env>, start_t>,
|
||||
__detail::__transform_sender_t<__detail::__completing_domain<_Sndr, _Env>, set_value_t>>;
|
||||
|
||||
public:
|
||||
template <class _Sndr, class _Env, class _ImplFn = __impl_fn_t<_Sndr, _Env>>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr&& __sndr, const _Env& __env) const
|
||||
noexcept(noexcept(_ImplFn()(static_cast<_Sndr&&>(__sndr), __env)))
|
||||
-> decltype(_ImplFn()(static_cast<_Sndr&&>(__sndr), __env))
|
||||
{
|
||||
return _ImplFn()(static_cast<_Sndr&&>(__sndr), __env);
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT transform_sender_t transform_sender{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_TRANSFORM_SENDER
|
||||
@@ -1,47 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_TYPE_TRAITS
|
||||
#define __CUDAX_EXECUTION_TYPE_TRAITS
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__type_traits/type_list.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh> // IWYU pragma: export
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
template <class _Ret, class... _Args>
|
||||
using __fn_t _CCCL_NODEBUG_ALIAS = _Ret(_Args...);
|
||||
|
||||
template <class _Ret, class... _Args>
|
||||
using __fn_ptr_t _CCCL_NODEBUG_ALIAS = _Ret (*)(_Args...);
|
||||
|
||||
template <class _Ty>
|
||||
using __cref_t _CCCL_NODEBUG_ALIAS = _Ty const&;
|
||||
|
||||
using __cp _CCCL_NODEBUG_ALIAS = ::cuda::std::__type_self;
|
||||
using __cpclr _CCCL_NODEBUG_ALIAS = ::cuda::std::__type_quote1<__cref_t>;
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_TYPE_TRAITS
|
||||
@@ -1,434 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_UTILITY
|
||||
#define __CUDAX_EXECUTION_UTILITY
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__runtime/api_wrapper.h>
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__concepts/constructible.h>
|
||||
#include <cuda/std/__exception/cuda_error.h>
|
||||
#include <cuda/std/__host_stdlib/new>
|
||||
#include <cuda/std/__memory/unique_ptr.h>
|
||||
#include <cuda/std/__tuple_dir/ignore.h>
|
||||
#include <cuda/std/__type_traits/copy_cvref.h>
|
||||
#include <cuda/std/__type_traits/decay.h>
|
||||
#include <cuda/std/__type_traits/enable_if.h>
|
||||
#include <cuda/std/__type_traits/is_callable.h>
|
||||
#include <cuda/std/__type_traits/is_same.h>
|
||||
#include <cuda/std/__type_traits/is_void.h>
|
||||
#include <cuda/std/__type_traits/type_list.h>
|
||||
#include <cuda/std/__utility/pod_tuple.h>
|
||||
#include <cuda/std/initializer_list>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/meta.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
_CCCL_GLOBAL_CONSTANT size_t __npos = static_cast<size_t>(-1);
|
||||
|
||||
struct __empty
|
||||
{};
|
||||
|
||||
template <class...>
|
||||
struct [[deprecated]] __deprecated
|
||||
{};
|
||||
|
||||
struct __nil
|
||||
{};
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr auto __maximum(::cuda::std::initializer_list<size_t> __il) noexcept -> size_t
|
||||
{
|
||||
size_t __max = 0;
|
||||
for (auto i : __il)
|
||||
{
|
||||
if (i > __max)
|
||||
{
|
||||
__max = i;
|
||||
}
|
||||
}
|
||||
return __max;
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr auto __find_pos(bool const* const __begin, bool const* const __end) noexcept -> size_t
|
||||
{
|
||||
for (bool const* __where = __begin; __where != __end; ++__where)
|
||||
{
|
||||
if (*__where)
|
||||
{
|
||||
return static_cast<size_t>(__where - __begin);
|
||||
}
|
||||
}
|
||||
return __npos;
|
||||
}
|
||||
|
||||
template <class _Ty, class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto __index_of() noexcept -> size_t
|
||||
{
|
||||
constexpr bool __map[] = {__same_as<_Ty, _Ts>...};
|
||||
return execution::__find_pos(__map, __map + sizeof...(_Ts));
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Ty, class _Uy = _Ty>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto __exchange(_Ty& __obj, _Uy&& __new_value) noexcept -> _Ty
|
||||
{
|
||||
constexpr bool __is_nothrow = //
|
||||
noexcept(_Ty(static_cast<_Ty&&>(__obj))) && //
|
||||
noexcept(__obj = static_cast<_Uy&&>(__new_value)); //
|
||||
static_assert(__is_nothrow);
|
||||
|
||||
_Ty old_value = static_cast<_Ty&&>(__obj);
|
||||
__obj = static_cast<_Uy&&>(__new_value);
|
||||
return old_value;
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Ty>
|
||||
_CCCL_HOST_DEVICE_API constexpr void __swap(_Ty& __left, _Ty& __right) noexcept
|
||||
{
|
||||
constexpr bool __is_nothrow = //
|
||||
noexcept(_Ty(static_cast<_Ty&&>(__left))) && //
|
||||
noexcept(__left = static_cast<_Ty&&>(__right)); //
|
||||
static_assert(__is_nothrow);
|
||||
|
||||
_Ty __tmp = static_cast<_Ty&&>(__left);
|
||||
__left = static_cast<_Ty&&>(__right);
|
||||
__right = static_cast<_Ty&&>(__tmp);
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Ty>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __decay_copy(_Ty&& __ty) noexcept(__nothrow_decay_copyable<_Ty>)
|
||||
-> decay_t<_Ty>
|
||||
{
|
||||
return static_cast<_Ty&&>(__ty);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_API inline auto __get_pointer_attributes(const void* __pv) -> ::cudaPointerAttributes
|
||||
{
|
||||
::cudaPointerAttributes __attrs;
|
||||
_CCCL_TRY_CUDA_API(::cudaPointerGetAttributes, "cudaPointerGetAttributes failed", &__attrs, __pv);
|
||||
return __attrs;
|
||||
}
|
||||
|
||||
#define __debug_printf(...) (printf(__VA_ARGS__), [] NV_IF_ELSE_TARGET(NV_IS_HOST, (fflush(stdout);), (void(0);))())
|
||||
|
||||
// This function can only be called from a catch handler.
|
||||
[[nodiscard]] _CCCL_HOST_API inline auto __get_cuda_error_from_active_exception() -> ::cudaError_t
|
||||
{
|
||||
try
|
||||
{
|
||||
throw; // rethrow the active exception
|
||||
}
|
||||
catch (::cuda::cuda_error& __err)
|
||||
{
|
||||
return __err.status();
|
||||
}
|
||||
catch (::std::bad_alloc&)
|
||||
{
|
||||
return ::cudaErrorMemoryAllocation;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
return ::cudaErrorUnknown; // fallback if no cuda error is found
|
||||
}
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
template <class _Ty>
|
||||
struct __managed_box : private __immovable
|
||||
{
|
||||
using value_type = _Ty;
|
||||
|
||||
_CCCL_HIDE_FROM_ABI __managed_box() = default;
|
||||
|
||||
_CCCL_TEMPLATE(class... _Args)
|
||||
_CCCL_REQUIRES(::cuda::std::constructible_from<_Ty, _Args...>)
|
||||
_CCCL_HOST_API explicit __managed_box(_Args&&... __args) noexcept(__nothrow_constructible<_Ty, _Args...>)
|
||||
: __value{static_cast<_Args&&>(__args)...}
|
||||
{
|
||||
_CCCL_ASSERT(execution::__get_pointer_attributes(this).type == cudaMemoryTypeManaged,
|
||||
"__managed_box must be allocated in managed memory");
|
||||
}
|
||||
|
||||
template <class... _Args>
|
||||
_CCCL_HOST_API static auto __make_unique(_Args&&... __args) -> ::cuda::std::unique_ptr<__managed_box>
|
||||
{
|
||||
return ::cuda::std::make_unique<__managed_box>(static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
_CCCL_HOST_API static auto operator new(size_t __size) -> void*
|
||||
{
|
||||
void* __ptr = nullptr;
|
||||
_CCCL_TRY_CUDA_API(::cudaMallocManaged, "cudaMallocManaged failed", &__ptr, __size);
|
||||
::cuda::std::ignore = ::cudaDeviceSynchronize(); // Ensure the memory is allocated before returning it.
|
||||
return __ptr;
|
||||
}
|
||||
|
||||
_CCCL_HOST_API static void operator delete(void* __ptr, size_t) noexcept
|
||||
{
|
||||
::cuda::std::ignore = ::cudaDeviceSynchronize(); // Ensure all operations on the memory are complete.
|
||||
::cuda::std::ignore = ::cudaFree(__ptr);
|
||||
}
|
||||
|
||||
value_type __value;
|
||||
|
||||
private:
|
||||
// Prevent the construction of __managed_box without dynamic allocation.
|
||||
friend struct ::cuda::std::default_delete<__managed_box<_Ty>>;
|
||||
~__managed_box() = default;
|
||||
};
|
||||
|
||||
//! @brief A callable that wraps a set of functions and calls the first one that is
|
||||
//! callable with a given set of arguments.
|
||||
template <class... _Fns>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __first_callable
|
||||
{
|
||||
private:
|
||||
//! @brief Returns the first function that is callable with a given set of arguments.
|
||||
template <class... _Args, class _Self>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto __get_1st(_Self&& __self) noexcept -> decltype(auto)
|
||||
{
|
||||
// NOLINTNEXTLINE (modernize-avoid-c-arrays)
|
||||
constexpr bool __flags[] = {__callable<::cuda::std::__copy_cvref_t<_Self, _Fns>, _Args...>..., false};
|
||||
constexpr size_t __idx = execution::__find_pos(__flags, __flags + sizeof...(_Fns));
|
||||
if constexpr (__idx != __npos)
|
||||
{
|
||||
return ::cuda::std::__get<__idx>(static_cast<_Self&&>(__self).__fns_);
|
||||
}
|
||||
}
|
||||
|
||||
//! @brief Alias for the type of the first function that is callable with a given set of arguments.
|
||||
template <class _Self, class... _Args>
|
||||
using __1st_fn_t _CCCL_NODEBUG_ALIAS = decltype(__first_callable::__get_1st<_Args...>(declval<_Self>()));
|
||||
|
||||
public:
|
||||
//! @brief Calls the first function that is callable with a given set of arguments.
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class... _Args>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto
|
||||
operator()(_Args&&... __args) && noexcept(__nothrow_callable<__1st_fn_t<__first_callable, _Args...>, _Args...>)
|
||||
-> __call_result_t<__1st_fn_t<__first_callable, _Args...>, _Args...>
|
||||
{
|
||||
return __first_callable::__get_1st<_Args...>(static_cast<__first_callable&&>(*this))(
|
||||
static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
//! @overload
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class... _Args>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Args&&... __args) const& noexcept(
|
||||
__nothrow_callable<__1st_fn_t<__first_callable const&, _Args...>, _Args...>)
|
||||
-> __call_result_t<__1st_fn_t<__first_callable const&, _Args...>, _Args...>
|
||||
{
|
||||
return __first_callable::__get_1st<_Args...>(*this)(static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
::cuda::std::__tuple<_Fns...> __fns_;
|
||||
};
|
||||
|
||||
template <class... _Fns>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES __first_callable(_Fns...) -> __first_callable<_Fns...>;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// __call_or
|
||||
namespace __detail
|
||||
{
|
||||
// call a function with a set of arguments or return a default value if the function is
|
||||
// not callable.
|
||||
struct __call_or_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Fn, class _Default = __nil, class... _Args)
|
||||
_CCCL_REQUIRES(__callable<_Fn, _Args...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Fn&& __fn, _Default&&, _Args&&... __args) const
|
||||
noexcept(__nothrow_callable<_Fn, _Args...>) -> __call_result_t<_Fn, _Args...>
|
||||
{
|
||||
return static_cast<_Fn&&>(__fn)(static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Default = __nil,
|
||||
class _Result = ::cuda::std::_If<__same_as<_Default, __nil>, void, _Default>,
|
||||
class... _Args>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
operator()(::cuda::std::__ignore_t, _Default&& __default, _Args&&...) const noexcept(__nothrow_movable<_Default>)
|
||||
-> _Result
|
||||
{
|
||||
return static_cast<_Result>(static_cast<_Default&&>(__default));
|
||||
}
|
||||
};
|
||||
|
||||
template <bool>
|
||||
struct __call_result_or
|
||||
{
|
||||
template <class _Fn, class _Default, class... _Args>
|
||||
using __call _CCCL_NODEBUG_ALIAS = __call_result_t<__call_or_t, _Fn, _Default, _Args...>;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct __call_result_or<true>
|
||||
{
|
||||
template <class _Fn, class _Default, class... _Args>
|
||||
using __call _CCCL_NODEBUG_ALIAS = decltype(__call_or_t()(declval<_Fn>(), {}, declval<_Args>()...));
|
||||
};
|
||||
} // namespace __detail
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT __detail::__call_or_t __call_or{};
|
||||
|
||||
template <class _Fn, class _Default, class... _Args>
|
||||
using __call_result_or_t _CCCL_NODEBUG_ALIAS =
|
||||
cuda::std::__type_call<__detail::__call_result_or<__same_as<_Default, __nil>>, _Fn, _Default, _Args...>;
|
||||
|
||||
//! @brief A callable that always return a value of type _Ty, regardless of the arguments
|
||||
//! passed to it.
|
||||
template <class _Ty>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __always
|
||||
{
|
||||
template <class... _Args>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Args&&...) && noexcept -> _Ty&&
|
||||
{
|
||||
return static_cast<_Ty&&>(__value);
|
||||
}
|
||||
|
||||
template <class... _Args>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Args&&...) const& noexcept -> _Ty const&
|
||||
{
|
||||
return __value;
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ _Ty __value{};
|
||||
};
|
||||
|
||||
template <class _Ty>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES __always(_Ty) -> __always<_Ty>;
|
||||
|
||||
// @brief A type that turns a nullary callable into an object that is
|
||||
// implicitly convertible to the result type of the callable.
|
||||
template <class _Fn>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __emplace_from
|
||||
{
|
||||
using __result_t = __call_result_t<_Fn>;
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_HOST_DEVICE_API constexpr operator __result_t() && noexcept(__nothrow_callable<_Fn>)
|
||||
{
|
||||
return static_cast<_Fn&&>(__fn_)();
|
||||
}
|
||||
|
||||
_CCCL_NO_UNIQUE_ADDRESS _Fn __fn_;
|
||||
};
|
||||
|
||||
template <class _Fn>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES __emplace_from(_Fn) -> __emplace_from<_Fn>;
|
||||
|
||||
template <class _Ty, class... _Us>
|
||||
using __unless_one_of_t = ::cuda::std::enable_if_t<__none_of<_Ty, _Us...>, _Ty>;
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wnon-template-friend")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(probable_guiding_friend)
|
||||
_CCCL_BEGIN_NV_DIAG_SUPPRESS(probable_guiding_friend)
|
||||
|
||||
// __zip/__unzip is for keeping type names short. It has the unfortunate side
|
||||
// effect of obfuscating the types.
|
||||
namespace
|
||||
{
|
||||
template <size_t _Ny>
|
||||
struct __slot
|
||||
{
|
||||
friend constexpr auto __slot_allocated(__slot<_Ny>);
|
||||
};
|
||||
|
||||
template <class _Type, size_t _Ny>
|
||||
struct __allocate_slot
|
||||
{
|
||||
static constexpr size_t __value = _Ny;
|
||||
|
||||
friend constexpr auto __slot_allocated(__slot<_Ny>)
|
||||
{
|
||||
return static_cast<_Type (*)()>(nullptr);
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Type, size_t _Id = 0, size_t _Pow2 = 0>
|
||||
constexpr auto __next(long) -> size_t;
|
||||
|
||||
// If __slot_allocated(__slot<_Id>) has NOT been defined, then SFINAE will keep
|
||||
// this function out of the overload set...
|
||||
template <class _Type, //
|
||||
size_t _Id = 0,
|
||||
size_t _Pow2 = 0,
|
||||
bool = !__slot_allocated(__slot<_Id + (1 << _Pow2) - 1>())>
|
||||
constexpr auto __next(int) -> size_t
|
||||
{
|
||||
return execution::__next<_Type, _Id, _Pow2 + 1>(0);
|
||||
}
|
||||
|
||||
template <class _Type, size_t _Id, size_t _Pow2>
|
||||
constexpr auto __next(long) -> size_t
|
||||
{
|
||||
if constexpr (_Pow2 == 0)
|
||||
{
|
||||
return __allocate_slot<_Type, _Id>::__value;
|
||||
}
|
||||
else
|
||||
{
|
||||
return execution::__next<_Type, _Id + (1 << (_Pow2 - 1)), 0>(0);
|
||||
}
|
||||
}
|
||||
|
||||
// Prior to Clang 12, we can't use the __slot trick to erase long type names
|
||||
// because of a compiler bug. We'll just use the original type name in that case.
|
||||
#if _CCCL_COMPILER(CLANG, <, 12)
|
||||
|
||||
template <class _Type>
|
||||
using __zip _CCCL_NODEBUG_ALIAS = _Type;
|
||||
|
||||
template <class _Id>
|
||||
using __unzip _CCCL_NODEBUG_ALIAS = _Id;
|
||||
|
||||
#else // ^^^ _CCCL_COMPILER(CLANG, <, 12) ^^^ / vvv !_CCCL_COMPILER(CLANG, <, 12) vvv
|
||||
|
||||
template <class _Type, size_t _Val = execution::__next<_Type>(0)>
|
||||
using __zip _CCCL_NODEBUG_ALIAS = __slot<_Val>;
|
||||
|
||||
template <class _Id>
|
||||
using __unzip _CCCL_NODEBUG_ALIAS = decltype(__slot_allocated(_Id())());
|
||||
|
||||
#endif // ^^^ !_CCCL_COMPILER(CLANG, <, 12) ^^^
|
||||
|
||||
// burn the first slot
|
||||
using __ignore_this_typedef [[maybe_unused]] = __zip<void>;
|
||||
} // namespace
|
||||
|
||||
_CCCL_END_NV_DIAG_SUPPRESS()
|
||||
_CCCL_DIAG_POP
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_UTILITY
|
||||
@@ -1,278 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_VARIANT
|
||||
#define __CUDAX_EXECUTION_VARIANT
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__cccl/assert.h>
|
||||
#include <cuda/std/__concepts/constructible.h>
|
||||
#include <cuda/std/__memory/addressof.h>
|
||||
#include <cuda/std/__memory/construct_at.h>
|
||||
#include <cuda/std/__new/device_new.h>
|
||||
#include <cuda/std/__new/launder.h>
|
||||
#include <cuda/std/__type_traits/copy_cvref.h>
|
||||
#include <cuda/std/__type_traits/decay.h>
|
||||
#include <cuda/std/__type_traits/remove_reference.h>
|
||||
#include <cuda/std/__type_traits/type_list.h>
|
||||
#include <cuda/std/__utility/declval.h>
|
||||
#include <cuda/std/__utility/exchange.h>
|
||||
#include <cuda/std/__utility/integer_sequence.h>
|
||||
#include <cuda/std/__utility/monostate.h>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/meta.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
|
||||
#include <exception> // IWYU pragma: keep
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
/********************************************************************************/
|
||||
/* NB: The variant type implemented here default-constructs into the valueless */
|
||||
/* state. This is different from std::variant which default-constructs into the */
|
||||
/* first alternative. This is done to simplify the implementation and to avoid */
|
||||
/* the need for a default constructor for each alternative type. */
|
||||
/********************************************************************************/
|
||||
|
||||
using __monostate = ::cuda::std::monostate;
|
||||
|
||||
template <size_t _Idx, bool _Check = true, class _CvVariant>
|
||||
[[nodiscard]]
|
||||
_CCCL_TRIVIAL_API constexpr auto&& __variant_get(_CvVariant&& __var) noexcept
|
||||
{
|
||||
using __variant_t = ::cuda::std::remove_reference_t<_CvVariant>;
|
||||
using __element_t = typename __variant_t::template __at<_Idx>;
|
||||
using __result_t = ::cuda::std::__copy_cvref_t<_CvVariant, __element_t>;
|
||||
if constexpr (_Check)
|
||||
{
|
||||
_CCCL_ASSERT(__var.__index() == _Idx, "variant index mismatch");
|
||||
}
|
||||
return static_cast<__result_t&&>(*static_cast<__element_t*>(__var.__ptr()));
|
||||
}
|
||||
|
||||
struct __visit_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <size_t... _Idx, class _Fn, class _CvVariant, class... _As>
|
||||
_CCCL_HOST_DEVICE_API static void
|
||||
__visit(::cuda::std::index_sequence<_Idx...>*,
|
||||
const size_t __index,
|
||||
_Fn&& __fn,
|
||||
_CvVariant&& __var,
|
||||
_As&&... __as) //
|
||||
noexcept(noexcept((
|
||||
(_Idx == __index ? declval<_Fn>()(declval<_As>()..., execution::__variant_get<_Idx, false>(declval<_CvVariant>()))
|
||||
: void()),
|
||||
...)))
|
||||
{
|
||||
_CCCL_ASSERT(__index != __npos, "cannot visit a stateless variant");
|
||||
// Use a fold expression to avoid the need for a loop.
|
||||
((_Idx == __index
|
||||
? static_cast<_Fn&&>(
|
||||
__fn)(static_cast<_As&&>(__as)..., execution::__variant_get<_Idx, false>(static_cast<_CvVariant&&>(__var)))
|
||||
: void()),
|
||||
...);
|
||||
}
|
||||
|
||||
template <class _Fn, class _CvVariant, class... _As>
|
||||
_CCCL_TRIVIAL_API void operator()(_Fn&& __fn, _CvVariant&& __var, _As&&... __as) const noexcept(
|
||||
noexcept(__visit_t::__visit(__var.__indices(), size_t(), declval<_Fn>(), declval<_CvVariant>(), declval<_As>()...)))
|
||||
{
|
||||
__visit_t::__visit(
|
||||
__var.__indices(),
|
||||
__var.__index(),
|
||||
static_cast<_Fn&&>(__fn),
|
||||
static_cast<_CvVariant&&>(__var),
|
||||
static_cast<_As&&>(__as)...);
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT __visit_t __visit{};
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
struct __destroy_fn
|
||||
{
|
||||
template <class _Ty>
|
||||
_CCCL_HOST_DEVICE_API void operator()(_Ty& __ty) const noexcept
|
||||
{
|
||||
::cuda::std::__destroy_at(::cuda::std::addressof(__ty));
|
||||
}
|
||||
};
|
||||
|
||||
struct __move_to_fn
|
||||
{
|
||||
template <class _Ty>
|
||||
_CCCL_HOST_DEVICE_API void operator()(_Ty&& __from) const noexcept
|
||||
{
|
||||
::cuda::std::__construct_at(static_cast<decay_t<_Ty>*>(__to), static_cast<_Ty&&>(__from));
|
||||
}
|
||||
|
||||
void* __to;
|
||||
};
|
||||
} // namespace __detail
|
||||
|
||||
template <class... _Ts>
|
||||
class __variant
|
||||
{
|
||||
public:
|
||||
template <size_t _Ny>
|
||||
using __at _CCCL_NODEBUG_ALIAS = ::cuda::std::__type_index_c<_Ny, _Ts...>;
|
||||
|
||||
_CCCL_HOST_DEVICE_API __variant() noexcept {}
|
||||
|
||||
_CCCL_TEMPLATE(class...)
|
||||
_CCCL_REQUIRES((::cuda::std::move_constructible<_Ts> && ...))
|
||||
__variant(__variant&& __other) noexcept
|
||||
{
|
||||
if (__other.__index_ != __npos)
|
||||
{
|
||||
__visit_t::__visit(
|
||||
__indices(), __other.__index(), __detail::__move_to_fn{__ptr()}, static_cast<__variant&&>(__other));
|
||||
__index_ = __other.__index_;
|
||||
__other.__reset();
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API ~__variant()
|
||||
{
|
||||
__reset();
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_TRIVIAL_API void* __ptr() noexcept
|
||||
{
|
||||
return __storage_;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_TRIVIAL_API size_t __index() const noexcept
|
||||
{
|
||||
return __index_;
|
||||
}
|
||||
|
||||
template <int = 0, class _Ty>
|
||||
_CCCL_HOST_DEVICE_API auto __emplace(_Ty&& __value) noexcept(__nothrow_decay_copyable<_Ty>) -> decay_t<_Ty>&
|
||||
{
|
||||
return __emplace<decay_t<_Ty>>(static_cast<_Ty&&>(__value));
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Ty, class... _As>
|
||||
_CCCL_HOST_DEVICE_API auto __emplace(_As&&... __as) noexcept(__nothrow_constructible<_Ty, _As...>) -> _Ty&
|
||||
{
|
||||
constexpr size_t __new_index = __index_of<_Ty, _Ts...>();
|
||||
static_assert(__new_index != __npos, "Type not in variant");
|
||||
|
||||
__reset();
|
||||
_Ty* __value = ::new (__ptr()) _Ty{static_cast<_As&&>(__as)...};
|
||||
__index_ = __new_index;
|
||||
return *::cuda::std::launder(__value);
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <size_t _Ny, class... _As>
|
||||
_CCCL_HOST_DEVICE_API auto __emplace_at(_As&&... __as) noexcept(__nothrow_constructible<__at<_Ny>, _As...>)
|
||||
-> __at<_Ny>&
|
||||
{
|
||||
static_assert(_Ny < sizeof...(_Ts), "variant index is too large");
|
||||
|
||||
__reset();
|
||||
__at<_Ny>* __value = ::new (__ptr()) __at<_Ny>{static_cast<_As&&>(__as)...};
|
||||
__index_ = _Ny;
|
||||
return *::cuda::std::launder(__value);
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
template <class _Fn, class... _As>
|
||||
_CCCL_HOST_DEVICE_API auto __emplace_from(_Fn&& __fn, _As&&... __as) //
|
||||
noexcept(__nothrow_callable<_Fn, _As...>) -> __call_result_t<_Fn, _As...>&
|
||||
{
|
||||
using __result_t _CCCL_NODEBUG_ALIAS = __call_result_t<_Fn, _As...>;
|
||||
constexpr size_t __new_index = __index_of<__result_t, _Ts...>();
|
||||
static_assert(__new_index != __npos, "_Type not in variant");
|
||||
|
||||
__reset();
|
||||
__result_t* __value = ::new (__ptr()) __result_t(static_cast<_Fn&&>(__fn)(static_cast<_As&&>(__as)...));
|
||||
__index_ = __new_index;
|
||||
return *::cuda::std::launder(__value);
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void __reset() noexcept
|
||||
{
|
||||
if (__index_ != __npos)
|
||||
{
|
||||
// We must set the index to __npos *before* destroying the value on the off chance that
|
||||
// destroying the active value might cause the destruction of *this. But then, we must
|
||||
// tell the __visit function what the old index was so it can destroy the correct type.
|
||||
const auto __index = ::cuda::std::exchange(__index_, __npos);
|
||||
__visit_t::__visit(__indices(), __index, __detail::__destroy_fn{}, *this);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
friend struct __visit_t;
|
||||
|
||||
template <size_t, bool _Check, class _CvVariant>
|
||||
friend _CCCL_HOST_DEVICE_API constexpr auto&& __variant_get(_CvVariant&& __var) noexcept;
|
||||
|
||||
_CCCL_TRIVIAL_API static constexpr auto __indices() noexcept -> ::cuda::std::make_index_sequence<sizeof...(_Ts)>*
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
size_t __index_{__npos};
|
||||
alignas(_Ts...) unsigned char __storage_[__maximum({sizeof(_Ts)...})];
|
||||
};
|
||||
|
||||
template <>
|
||||
class __variant<>
|
||||
{
|
||||
public:
|
||||
_CCCL_HIDE_FROM_ABI __variant() noexcept = default;
|
||||
|
||||
[[nodiscard]] _CCCL_TRIVIAL_API static constexpr size_t __index() noexcept
|
||||
{
|
||||
return __npos;
|
||||
}
|
||||
|
||||
private:
|
||||
friend struct __visit_t;
|
||||
|
||||
_CCCL_TRIVIAL_API static constexpr auto __indices() noexcept -> ::cuda::std::index_sequence<>*
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
};
|
||||
|
||||
template <class... _Ts>
|
||||
using __nullable_variant _CCCL_NODEBUG_ALIAS = __variant<__monostate, _Ts...>;
|
||||
|
||||
template <class... _Ts>
|
||||
using __decayed_variant _CCCL_NODEBUG_ALIAS = __variant<decay_t<_Ts>...>;
|
||||
|
||||
template <class... _Ts>
|
||||
using __nullable_decayed_variant _CCCL_NODEBUG_ALIAS = __variant<__monostate, decay_t<_Ts>...>;
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_VARIANT
|
||||
@@ -1,210 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_VISIT
|
||||
#define __CUDAX_EXECUTION_VISIT
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__tuple_dir/ignore.h>
|
||||
#include <cuda/std/__type_traits/copy_cvref.h>
|
||||
#include <cuda/std/__type_traits/enable_if.h>
|
||||
#include <cuda/std/__type_traits/is_aggregate.h>
|
||||
#include <cuda/std/__utility/pod_tuple.h>
|
||||
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
#define _CCCL_BIND_CHILD(_Ord) , _CCCL_PP_CAT(__child, _Ord)
|
||||
#define _CCCL_FWD_CHILD(_Ord) , _CCCL_FWD_LIKE(_Sndr, _CCCL_PP_CAT(__child, _Ord))
|
||||
#define _CCCL_FWD_LIKE(_X, _Y) static_cast<::cuda::std::__copy_cvref_t<_X&&, decltype(_Y)>>(_Y)
|
||||
|
||||
#if _CCCL_HAS_BUILTIN(__builtin_structured_binding_size)
|
||||
# define _CCCL_BUILTIN_STRUCTURED_BINDING_SIZE(...) __builtin_structured_binding_size(__VA_ARGS__)
|
||||
#endif // _CCCL_HAS_BUILTIN(__builtin_structured_binding_size)
|
||||
|
||||
#if _CCCL_CUDA_COMPILER(NVCC)
|
||||
# undef _CCCL_BUILTIN_STRUCTURED_BINDING_SIZE
|
||||
#endif // _CCCL_CUDA_COMPILER(NVCC)
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
#if defined(_CCCL_BUILTIN_STRUCTURED_BINDING_SIZE)
|
||||
|
||||
# if _CCCL_HAS_CONCEPTS()
|
||||
|
||||
template <class _Sndr>
|
||||
inline constexpr int structured_binding_size = -1;
|
||||
|
||||
template <class _Sndr>
|
||||
requires(_CCCL_BUILTIN_STRUCTURED_BINDING_SIZE(_Sndr) >= 0)
|
||||
inline constexpr int structured_binding_size<_Sndr> = _CCCL_BUILTIN_STRUCTURED_BINDING_SIZE(_Sndr);
|
||||
|
||||
# else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / !_CCCL_HAS_CONCEPTS() vvv
|
||||
|
||||
template <class _Sndr, class _Enable = void>
|
||||
inline constexpr int __structured_binding_size_impl = -1;
|
||||
|
||||
template <class _Sndr>
|
||||
inline constexpr int
|
||||
__structured_binding_size_impl<_Sndr, ::cuda::std::enable_if_t<_CCCL_BUILTIN_STRUCTURED_BINDING_SIZE(_Sndr) >= 0>> =
|
||||
static_cast<int>(_CCCL_BUILTIN_STRUCTURED_BINDING_SIZE(_Sndr));
|
||||
|
||||
template <class _Sndr, class _Enable = void>
|
||||
inline constexpr int structured_binding_size = __structured_binding_size_impl<_Sndr>;
|
||||
|
||||
# endif // _CCCL_HAS_CONCEPTS()
|
||||
|
||||
#else // ^^^ _CCCL_BUILTIN_STRUCTURED_BINDING_SIZE ^^^ / vvv !_CCCL_BUILTIN_STRUCTURED_BINDING_SIZE vvv
|
||||
|
||||
struct __any_t
|
||||
{
|
||||
template <class _Ty>
|
||||
_CCCL_HOST_DEVICE_API operator _Ty&&();
|
||||
};
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wmissing-field-initializers")
|
||||
|
||||
// use the "magic tuple" trick to get the arity of a structured binding
|
||||
// see https://github.com/apolukhin/magic_get
|
||||
template <class _Ty, bool = ::cuda::std::is_aggregate_v<_Ty>>
|
||||
struct __arity_of_t
|
||||
{
|
||||
template <class... _Ts, class _Uy = _Ty, class _Uy2 = decltype(_Uy{_Ts{}...}), class _Self = __arity_of_t>
|
||||
_CCCL_HOST_DEVICE_API auto operator()(_Ts... __ts) -> decltype(_Self{}(__ts..., __any_t{}));
|
||||
|
||||
template <class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API auto operator()(_Ts...) const -> char (*)[sizeof...(_Ts) + 1];
|
||||
};
|
||||
|
||||
template <class _Ty>
|
||||
struct __arity_of_t<_Ty, false>
|
||||
{
|
||||
_CCCL_HOST_DEVICE_API auto operator()() const -> char*;
|
||||
};
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
// Specialize this for each sender type that can be used to initialize a structured binding.
|
||||
template <class _Sndr>
|
||||
inline constexpr int structured_binding_size = static_cast<int>(sizeof(*__arity_of_t<_Sndr>{}())) - 2;
|
||||
|
||||
#endif // ^^^ !_CCCL_BUILTIN_STRUCTURED_BINDING_SIZE ^^^
|
||||
|
||||
template <class _Sndr>
|
||||
inline constexpr int structured_binding_size<_Sndr&> = structured_binding_size<_Sndr>;
|
||||
|
||||
template <class _Sndr>
|
||||
inline constexpr int structured_binding_size<_Sndr const&> = structured_binding_size<_Sndr>;
|
||||
|
||||
// If structured bindings can be used to introduce a pack, then `visit` has a very simple
|
||||
// implementation.
|
||||
#if _CCCL_HAS_STRUCTURED_BINDINGS_PACK()
|
||||
|
||||
// C++26, structured binding can introduce a pack.
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT visit_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Visitor, class _CvSndr, class _Context)
|
||||
_CCCL_REQUIRES((structured_binding_size<_CvSndr> >= 2))
|
||||
_CCCL_NODEBUG_API constexpr auto operator()(_Visitor& __visitor, _CvSndr&& __sndr, _Context& __context) const
|
||||
-> decltype(auto)
|
||||
{
|
||||
auto&& [__tag, __data, ... __children] = static_cast<_CvSndr&&>(__sndr);
|
||||
return __visitor(__context, __tag, _CCCL_FWD_LIKE(_CvSndr, __data), _CCCL_FWD_LIKE(_CvSndr, __children)...);
|
||||
}
|
||||
};
|
||||
|
||||
#else // ^^^ __cpp_structured_bindings >= 202411L / !__cpp_structured_bindings >= 202411L vvv
|
||||
|
||||
// When structured bindings cannot introduce a pack, we need to manually unroll for a
|
||||
// fixed maximum arity.
|
||||
template <size_t _Arity>
|
||||
struct __sender_type_cannot_be_used_to_initialize_a_structured_binding;
|
||||
|
||||
template <size_t _Arity>
|
||||
struct __unpack
|
||||
{
|
||||
// This is to generate a compile-time error if the sender type cannot be used to
|
||||
// initialize a structured binding.
|
||||
_CCCL_HOST_DEVICE_API void operator()(::cuda::std::__ignore_t,
|
||||
__sender_type_cannot_be_used_to_initialize_a_structured_binding<_Arity>,
|
||||
::cuda::std::__ignore_t) const;
|
||||
};
|
||||
|
||||
# define _CCCL_UNPACK_SENDER(_Arity) \
|
||||
template <> \
|
||||
struct __unpack<2 + _Arity> \
|
||||
{ \
|
||||
_CCCL_EXEC_CHECK_DISABLE \
|
||||
template <class _Visitor, class _Sndr, class _Context> \
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Visitor& __visitor, _Sndr&& __sndr, _Context& __context) const \
|
||||
-> decltype(auto) \
|
||||
{ \
|
||||
auto&& [__tag, __data _CCCL_PP_REPEAT(_Arity, _CCCL_BIND_CHILD)] = static_cast<_Sndr&&>(__sndr); \
|
||||
return __visitor(__context, __tag, _CCCL_FWD_LIKE(_Sndr, __data) _CCCL_PP_REPEAT(_Arity, _CCCL_FWD_CHILD)); \
|
||||
} \
|
||||
}
|
||||
|
||||
_CCCL_UNPACK_SENDER(0);
|
||||
_CCCL_UNPACK_SENDER(1);
|
||||
_CCCL_UNPACK_SENDER(2);
|
||||
_CCCL_UNPACK_SENDER(3);
|
||||
_CCCL_UNPACK_SENDER(4);
|
||||
_CCCL_UNPACK_SENDER(5);
|
||||
_CCCL_UNPACK_SENDER(6);
|
||||
_CCCL_UNPACK_SENDER(7);
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT visit_t
|
||||
{
|
||||
_CCCL_TEMPLATE(class _Visitor, class _Sndr, class _Context)
|
||||
_CCCL_REQUIRES((structured_binding_size<_Sndr> >= 2))
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Visitor& __visitor, _Sndr&& __sndr, _Context& __context) const
|
||||
-> decltype(auto)
|
||||
{
|
||||
// This `if constexpr` shouldn't be needed given the `requires` clause above. It is
|
||||
// here because nvcc 12.0 has a bug where the full signature of the function template
|
||||
// -- including the return type -- is instantiated before the `requires` clause is
|
||||
// checked.
|
||||
if constexpr (structured_binding_size<_Sndr> >= 2)
|
||||
{
|
||||
return __unpack<structured_binding_size<_Sndr>>{}(__visitor, static_cast<_Sndr&&>(__sndr), __context);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
#endif // ^^^ __cpp_structured_bindings < 202411L
|
||||
|
||||
[[maybe_unused]]
|
||||
_CCCL_GLOBAL_CONSTANT visit_t visit{};
|
||||
|
||||
template <class _Visitor, class _CvSndr, class _Context>
|
||||
using __visit_result_t _CCCL_NODEBUG_ALIAS =
|
||||
decltype(execution::visit(declval<_Visitor&>(), declval<_CvSndr>(), declval<_Context&>()));
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#undef _CCCL_FWD_LIKE
|
||||
#undef _CCCL_FWD_CHILD
|
||||
#undef _CCCL_BIND_CHILD
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_VISIT
|
||||
@@ -1,574 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_WHEN_ALL
|
||||
#define __CUDAX_EXECUTION_WHEN_ALL
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__utility/immovable.h>
|
||||
#include <cuda/std/__cccl/unreachable.h>
|
||||
#include <cuda/std/__exception/exception_macros.h>
|
||||
#include <cuda/std/__numeric/exclusive_scan.h>
|
||||
#include <cuda/std/__tuple_dir/ignore.h>
|
||||
#include <cuda/std/__type_traits/common_type.h>
|
||||
#include <cuda/std/__type_traits/decay.h>
|
||||
#include <cuda/std/__type_traits/type_identity.h>
|
||||
#include <cuda/std/__type_traits/type_list.h>
|
||||
#include <cuda/std/__type_traits/underlying_type.h>
|
||||
#include <cuda/std/__utility/integer_sequence.h>
|
||||
#include <cuda/std/__utility/pod_tuple.h>
|
||||
#include <cuda/std/atomic>
|
||||
|
||||
#include <cuda/experimental/__detail/type_traits.cuh>
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/concepts.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/exception.cuh>
|
||||
#include <cuda/experimental/__execution/lazy.cuh>
|
||||
#include <cuda/experimental/__execution/meta.cuh>
|
||||
#include <cuda/experimental/__execution/queries.cuh>
|
||||
#include <cuda/experimental/__execution/stop_token.cuh>
|
||||
#include <cuda/experimental/__execution/transform_completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/transform_sender.cuh>
|
||||
#include <cuda/experimental/__execution/type_traits.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__execution/variant.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT when_all_t
|
||||
{
|
||||
template <class... _Sndrs>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
_CUDAX_SEMI_PRIVATE :
|
||||
// Extract the first template parameter of the __state_t specialization.
|
||||
// The first template parameter is the receiver type.
|
||||
template <class _State>
|
||||
using __rcvr_from_state_t _CCCL_NODEBUG_ALIAS =
|
||||
::cuda::std::__type_apply<::cuda::std::__detail::__type_at_fn<0>, _State>;
|
||||
|
||||
// Returns the completion signatures of a child sender. Throws an exception if
|
||||
// the child sender has more than one set_value completion signature.
|
||||
template <class _Child, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto __child_completions();
|
||||
|
||||
// Merges the completion signatures of the child senders into a single set of
|
||||
// completion signatures for the when_all sender.
|
||||
template <class... _Completions>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto __merge_completions(_Completions...);
|
||||
|
||||
/// The receivers connected to the when_all's sub-operations expose this as
|
||||
/// their environment. Its `get_stop_token` query returns the token from
|
||||
/// when_all's stop source. All other queries are forwarded to the outer
|
||||
/// receiver's environment.
|
||||
template <class _StateZip>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __env_t
|
||||
{
|
||||
using __state_t _CCCL_NODEBUG_ALIAS = __unzip<_StateZip>;
|
||||
using __rcvr_t _CCCL_NODEBUG_ALIAS = __rcvr_from_state_t<__state_t>;
|
||||
|
||||
__state_t& __state_;
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_stop_token_t) const noexcept -> inplace_stop_token
|
||||
{
|
||||
return __state_.__stop_token_;
|
||||
}
|
||||
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Query, class... _Args)
|
||||
_CCCL_REQUIRES(__forwarding_query<_Query> _CCCL_AND __queryable_with<env_of_t<__rcvr_t>, _Query, _Args...>)
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(_Query, _Args&&... __args) const
|
||||
noexcept(__nothrow_queryable_with<env_of_t<__rcvr_t>, _Query, _Args...>)
|
||||
-> __query_result_t<env_of_t<__rcvr_t>, _Query, _Args...>
|
||||
{
|
||||
return execution::get_env(__state_.__rcvr_).query(_Query{}, static_cast<_Args&&>(__args)...);
|
||||
}
|
||||
};
|
||||
|
||||
template <class _StateZip, size_t _Index>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_t
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
using __state_t _CCCL_NODEBUG_ALIAS = __unzip<_StateZip>;
|
||||
|
||||
__state_t& __state_;
|
||||
|
||||
template <class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_value(_Ts&&... __ts) noexcept
|
||||
{
|
||||
constexpr ::cuda::std::index_sequence_for<_Ts...>* idx = nullptr;
|
||||
__state_.template __set_value<_Index>(idx, static_cast<_Ts&&>(__ts)...);
|
||||
__state_.__arrive();
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_error(_Error&& __error) noexcept
|
||||
{
|
||||
__state_.__set_error(static_cast<_Error&&>(__error));
|
||||
__state_.__arrive();
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_stopped() noexcept
|
||||
{
|
||||
__state_.__set_stopped();
|
||||
__state_.__arrive();
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __env_t<_StateZip>
|
||||
{
|
||||
return {__state_};
|
||||
}
|
||||
};
|
||||
|
||||
enum __estate_t : int
|
||||
{
|
||||
__started,
|
||||
__error,
|
||||
__stopped
|
||||
};
|
||||
|
||||
/// @brief The data stored in the operation state and referred to
|
||||
/// by the receiver.
|
||||
/// @tparam _Rcvr The receiver connected to the when_all sender.
|
||||
/// @tparam _CvFn A metafunction to apply cv- and ref-qualifiers to the senders
|
||||
/// @tparam _Sndrs A tuple of the when_all sender's child senders.
|
||||
template <class _Rcvr, class _CvFn, class _Sndrs>
|
||||
struct __state_t;
|
||||
|
||||
template <class _Rcvr, class _CvFn, class _Ign0, class _Ign1, class... _Sndrs>
|
||||
struct __state_t<_Rcvr, _CvFn, ::cuda::std::__tuple<_Ign0, _Ign1, _Sndrs...>>
|
||||
{
|
||||
using __env_t _CCCL_NODEBUG_ALIAS = when_all_t::__env_t<__zip<__state_t>>;
|
||||
using __sndr_t _CCCL_NODEBUG_ALIAS = when_all_t::__sndr_t<_Sndrs...>;
|
||||
using __cv_sndr_t _CCCL_NODEBUG_ALIAS = ::cuda::std::__type_call1<_CvFn, __sndr_t>;
|
||||
|
||||
static constexpr auto __completions_and_offsets =
|
||||
__sndr_t::template __get_completions_and_offsets<__cv_sndr_t, __env_t>();
|
||||
|
||||
using __completions_t _CCCL_NODEBUG_ALIAS = decltype(__completions_and_offsets.first);
|
||||
using __values_t _CCCL_NODEBUG_ALIAS = __value_types<__completions_t, __lazy_tuple, __type_self_or<__nil>::__call>;
|
||||
using __errors_t _CCCL_NODEBUG_ALIAS = __error_types<__completions_t, __variant>;
|
||||
using __stop_tok_t _CCCL_NODEBUG_ALIAS = stop_token_of_t<env_of_t<_Rcvr>>;
|
||||
using __stop_callback_t _CCCL_NODEBUG_ALIAS = stop_callback_for_t<__stop_tok_t, __on_stop_request>;
|
||||
|
||||
_CCCL_HOST_DEVICE_API explicit __state_t(_Rcvr __rcvr, size_t __count)
|
||||
: __rcvr_{static_cast<_Rcvr&&>(__rcvr)}
|
||||
, __count_{__count}
|
||||
, __stop_source_{}
|
||||
, __stop_token_{__stop_source_.get_token()}
|
||||
, __state_{__started}
|
||||
, __errors_{}
|
||||
, __values_{}
|
||||
, __on_stop_{}
|
||||
{}
|
||||
|
||||
template <size_t _Index, size_t... _Jdx, class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API void
|
||||
__set_value(::cuda::std::index_sequence<_Jdx...>*, [[maybe_unused]] _Ts&&... __ts) noexcept
|
||||
{
|
||||
if constexpr (!__same_as<__values_t, __nil>)
|
||||
{
|
||||
constexpr size_t _Offset = __completions_and_offsets.second[_Index];
|
||||
if constexpr (__nothrow_decay_copyable<_Ts...>)
|
||||
{
|
||||
(__values_.template __emplace<_Jdx + _Offset>(static_cast<_Ts&&>(__ts)), ...);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
(__values_.template __emplace<_Jdx + _Offset>(static_cast<_Ts&&>(__ts)), ...);
|
||||
}
|
||||
_CCCL_CATCH_ALL
|
||||
{
|
||||
__set_error(execution::current_exception());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API void __set_error(_Error&& __err) noexcept
|
||||
{
|
||||
// TODO: Use weaker memory orders
|
||||
if (__error != __state_.exchange(__error))
|
||||
{
|
||||
__stop_source_.request_stop();
|
||||
// We won the race, free to write the error into the operation state
|
||||
// without worry.
|
||||
if constexpr (__nothrow_decay_copyable<_Error>)
|
||||
{
|
||||
__errors_.template __emplace<decay_t<_Error>>(static_cast<_Error&&>(__err));
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
__errors_.template __emplace<decay_t<_Error>>(static_cast<_Error&&>(__err));
|
||||
}
|
||||
_CCCL_CATCH_ALL
|
||||
{
|
||||
__errors_.template __emplace<exception_ptr>(execution::current_exception());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void __set_stopped() noexcept
|
||||
{
|
||||
::cuda::std::underlying_type_t<__estate_t> __expected = __started;
|
||||
// Transition to the "stopped" state if and only if we're in the
|
||||
// "started" state. (If this fails, it's because we're in an
|
||||
// error state, which trumps cancellation.)
|
||||
if (__state_.compare_exchange_strong(
|
||||
__expected, static_cast<::cuda::std::underlying_type_t<__estate_t>>(__stopped)))
|
||||
{
|
||||
__stop_source_.request_stop();
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void __arrive() noexcept
|
||||
{
|
||||
if (0 == --__count_)
|
||||
{
|
||||
__complete();
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API void __complete() noexcept
|
||||
{
|
||||
// Stop callback is no longer needed. Destroy it.
|
||||
__on_stop_.__destroy();
|
||||
// All child operations have completed and arrived at the barrier.
|
||||
switch (__state_.load(::cuda::std::memory_order_relaxed))
|
||||
{
|
||||
case __started:
|
||||
if constexpr (!__same_as<__values_t, __nil>)
|
||||
{
|
||||
// All child operations completed successfully:
|
||||
__values_.__apply(execution::set_value, static_cast<__values_t&&>(__values_), static_cast<_Rcvr&&>(__rcvr_));
|
||||
}
|
||||
break;
|
||||
case __error:
|
||||
// One or more child operations completed with an error:
|
||||
__visit(execution::set_error, static_cast<__errors_t&&>(__errors_), static_cast<_Rcvr&&>(__rcvr_));
|
||||
break;
|
||||
case __stopped:
|
||||
execution::set_stopped(static_cast<_Rcvr&&>(__rcvr_));
|
||||
break;
|
||||
default:;
|
||||
}
|
||||
}
|
||||
|
||||
_Rcvr __rcvr_;
|
||||
::cuda::std::atomic<size_t> __count_;
|
||||
inplace_stop_source __stop_source_;
|
||||
inplace_stop_token __stop_token_;
|
||||
::cuda::std::atomic<::cuda::std::underlying_type_t<__estate_t>> __state_;
|
||||
__errors_t __errors_;
|
||||
__values_t __values_;
|
||||
__lazy<__stop_callback_t> __on_stop_;
|
||||
};
|
||||
|
||||
struct __start_all
|
||||
{
|
||||
template <class... _Ops>
|
||||
_CCCL_HOST_DEVICE_API void operator()(_Ops&... __ops) const noexcept
|
||||
{
|
||||
(execution::start(__ops), ...);
|
||||
}
|
||||
};
|
||||
|
||||
/// The operation state for when_all
|
||||
template <class _Rcvr,
|
||||
class _CvFn,
|
||||
class _Sndrs,
|
||||
class = ::cuda::std::make_index_sequence<::cuda::std::__tuple_size_v<_Sndrs> - 2>>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t;
|
||||
|
||||
template <class _Rcvr, class _CvFn, size_t... _Idx, class _Ign0, class _Ign1, class... _Sndrs>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT
|
||||
__opstate_t<_Rcvr, _CvFn, ::cuda::std::__tuple<_Ign0, _Ign1, _Sndrs...>, ::cuda::std::index_sequence<_Idx...>>
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
using __sndrs_t _CCCL_NODEBUG_ALIAS =
|
||||
::cuda::std::__type_call<_CvFn, ::cuda::std::__tuple<_Ign0, _Ign1, _Sndrs...>>;
|
||||
using __state_t _CCCL_NODEBUG_ALIAS =
|
||||
when_all_t::__state_t<_Rcvr, _CvFn, ::cuda::std::__tuple<_Ign0, _Ign1, _Sndrs...>>;
|
||||
|
||||
// This function object is used to connect all the sub-operations with
|
||||
// receivers, each of which knows which elements in the values tuple it
|
||||
// is responsible for setting.
|
||||
struct __connect_subs_fn
|
||||
{
|
||||
template <class... _CvSndrs>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto
|
||||
operator()(__state_t& __state, ::cuda::std::__ignore_t, ::cuda::std::__ignore_t, _CvSndrs&&... __sndrs_) const
|
||||
{
|
||||
using __state_ref_t _CCCL_NODEBUG_ALIAS = __zip<__state_t>;
|
||||
// When there are no offsets, the when_all sender has no value
|
||||
// completions. All child senders can be connected to receivers
|
||||
// of the same type, saving template instantiations.
|
||||
[[maybe_unused]] constexpr bool __no_values =
|
||||
__same_as<decltype(__state_t::__completions_and_offsets.second), __nil>;
|
||||
// The offsets are used to determine which elements in the values
|
||||
// tuple each receiver is responsible for setting.
|
||||
return ::cuda::std::__tuple{execution::connect(
|
||||
static_cast<_CvSndrs&&>(__sndrs_), __rcvr_t<__state_ref_t, __no_values ? 0 : _Idx>{__state})...};
|
||||
}
|
||||
};
|
||||
|
||||
// This is a tuple of operation states for the sub-operations.
|
||||
using __sub_opstates_t _CCCL_NODEBUG_ALIAS =
|
||||
::cuda::std::__apply_result_t<__connect_subs_fn, __sndrs_t, __state_t&>;
|
||||
|
||||
__state_t __state_;
|
||||
__sub_opstates_t __sub_ops_;
|
||||
|
||||
/// Initialize the data member, connect all the sub-operations and
|
||||
/// save the resulting operation states in __sub_ops_.
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __opstate_t(__sndrs_t&& __sndrs_, _Rcvr __rcvr)
|
||||
: __state_{static_cast<_Rcvr&&>(__rcvr), sizeof...(_Sndrs)}
|
||||
, __sub_ops_{::cuda::std::__apply(__connect_subs_fn(), static_cast<__sndrs_t&&>(__sndrs_), __state_)}
|
||||
{}
|
||||
|
||||
_CCCL_IMMOVABLE(__opstate_t);
|
||||
|
||||
/// Start all the sub-operations.
|
||||
_CCCL_HOST_DEVICE_API constexpr void start() noexcept
|
||||
{
|
||||
// register stop callback:
|
||||
__state_.__on_stop_.__construct(
|
||||
get_stop_token(execution::get_env(__state_.__rcvr_)), __on_stop_request{__state_.__stop_source_});
|
||||
|
||||
if (__state_.__stop_source_.stop_requested())
|
||||
{
|
||||
// Manually clean up the stop callback. We won't be starting the
|
||||
// sub-operations, so they won't complete and clean up for us.
|
||||
__state_.__on_stop_.__destroy();
|
||||
|
||||
// Stop has already been requested. Don't bother starting the child
|
||||
// operations.
|
||||
execution::set_stopped(static_cast<_Rcvr&&>(__state_.__rcvr_));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Start all the sub-operations.
|
||||
::cuda::std::__apply(__start_all{}, __sub_ops_);
|
||||
|
||||
// If there are no sub-operations, we're done.
|
||||
if constexpr (sizeof...(_Sndrs) == 0)
|
||||
{
|
||||
__state_.__complete();
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class... _Ts>
|
||||
using __decay_all _CCCL_NODEBUG_ALIAS = ::cuda::std::__type_list<decay_t<_Ts>...>;
|
||||
|
||||
public:
|
||||
template <class... _Sndrs>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndrs... __sndrs) const;
|
||||
};
|
||||
|
||||
template <class _Child, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto when_all_t::__child_completions()
|
||||
{
|
||||
using __env_t _CCCL_NODEBUG_ALIAS = prop<get_stop_token_t, inplace_stop_token>;
|
||||
_CUDAX_LET_COMPLETIONS(auto(__completions) = get_completion_signatures<_Child, env<__env_t, __fwd_env_t<_Env>>...>())
|
||||
{
|
||||
if constexpr (__completions.count(set_value) > 1)
|
||||
{
|
||||
return invalid_completion_signature<_WHERE(_IN_ALGORITHM, when_all_t),
|
||||
_WHAT(_SENDER_HAS_TOO_MANY_SUCCESS_COMPLETIONS),
|
||||
_WITH_SENDER(_Child)>();
|
||||
}
|
||||
else
|
||||
{
|
||||
return __completions;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wunused-value")
|
||||
|
||||
template <class... _Completions>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto when_all_t::__merge_completions(_Completions... __cs)
|
||||
{
|
||||
// Use _CUDAX_LET_COMPLETIONS to ensure all completions are valid:
|
||||
_CUDAX_LET_COMPLETIONS(auto(__tmp) = (completion_signatures{}, ..., __cs)) // NB: uses overloaded comma operator
|
||||
{
|
||||
::cuda::std::ignore = __tmp; // silence unused variable warning
|
||||
auto __non_value_completions = concat_completion_signatures(
|
||||
completion_signatures<set_stopped_t()>{},
|
||||
transform_completion_signatures(__cs, __swallow_transform{}, __decay_transform<set_error_t>{})...);
|
||||
|
||||
if constexpr (((0 == __cs.count(set_value)) || ...))
|
||||
{
|
||||
// at least one child sender has no value completions at all, so the
|
||||
// when_all will never complete with set_value. return just the error and
|
||||
// stopped completions.
|
||||
return ::cuda::std::__pair{__non_value_completions, __nil{}};
|
||||
}
|
||||
else
|
||||
{
|
||||
std::array<size_t, sizeof...(_Completions)> __offsets = {
|
||||
__value_types<_Completions, ::cuda::std::__type_list, ::cuda::std::__type_list_size>::value...};
|
||||
(void) ::cuda::std::exclusive_scan(__offsets.begin(), __offsets.end(), __offsets.begin(), std::size_t(0));
|
||||
|
||||
// All child senders have exactly one value completion signature, each of
|
||||
// which may have multiple arguments. Concatenate all the arguments into a
|
||||
// single set_value_t completion signature.
|
||||
using __values_t _CCCL_NODEBUG_ALIAS = ::cuda::std::__type_call< //
|
||||
__type_concat_into<__type_function<set_value_t>>, //
|
||||
__value_types<_Completions, __decay_all, ::cuda::std::__type_self_t>...>;
|
||||
// Add the value completion to the error and stopped completions.
|
||||
auto __local = __non_value_completions + completion_signatures<__values_t>();
|
||||
// Check if any of the values or errors are not nothrow decay-copyable.
|
||||
constexpr bool __all_nothrow_decay_copyable =
|
||||
(__value_types<_Completions, __nothrow_decay_copyable_t, ::cuda::std::type_identity_t>::value && ...);
|
||||
return ::cuda::std::__pair{__local + __eptr_completion_if<!__all_nothrow_decay_copyable>(), __offsets};
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
// The sender for when_all
|
||||
template <class... _Sndrs>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT when_all_t::__sndr_t
|
||||
: ::cuda::std::__tuple<when_all_t, ::cuda::std::__ignore_t, _Sndrs...>
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
using __sndrs_t _CCCL_NODEBUG_ALIAS = ::cuda::std::__tuple<when_all_t, ::cuda::std::__ignore_t, _Sndrs...>;
|
||||
|
||||
template <class _Self, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto __get_completions_and_offsets()
|
||||
{
|
||||
return __merge_completions(__child_completions<::cuda::std::__copy_cvref_t<_Self, _Sndrs>, _Env...>()...);
|
||||
}
|
||||
|
||||
template <class _Self, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
return __get_completions_and_offsets<_Self, _Env...>().first;
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) && -> __opstate_t<_Rcvr, __cp, __sndrs_t>
|
||||
{
|
||||
return __opstate_t<_Rcvr, __cp, __sndrs_t>(static_cast<__sndrs_t&&>(*this), static_cast<_Rcvr&&>(__rcvr));
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
connect(_Rcvr __rcvr) const& -> __opstate_t<_Rcvr, __cpclr, __sndrs_t>
|
||||
{
|
||||
return __opstate_t<_Rcvr, __cpclr, __sndrs_t>(static_cast<__sndrs_t const&>(*this), static_cast<_Rcvr&&>(__rcvr));
|
||||
}
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __attrs_t
|
||||
{
|
||||
template <class _Tag, class... _Env>
|
||||
using __when_all_domain_t = __common_domain_t<__completion_domain_of_t<set_value_t, _Sndrs, _Env...>...>;
|
||||
|
||||
template <class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_domain_t<set_value_t>, const _Env&...) const noexcept
|
||||
-> __when_all_domain_t<set_value_t, _Env...>;
|
||||
|
||||
template <class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_domain_t<set_error_t>, const _Env&...) const noexcept
|
||||
-> __common_domain_t<__when_all_domain_t<set_value_t, _Env...>,
|
||||
__when_all_domain_t<set_error_t, _Env...>,
|
||||
__when_all_domain_t<set_stopped_t, _Env...>>;
|
||||
|
||||
template <class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
query(get_completion_domain_t<set_stopped_t>, const _Env&...) const noexcept
|
||||
-> __common_domain_t<__when_all_domain_t<set_value_t, _Env...>, __when_all_domain_t<set_stopped_t, _Env...>>;
|
||||
|
||||
template <class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_completion_behavior_t, const _Env&...) const noexcept
|
||||
{
|
||||
return (execution::min) (execution::get_completion_behavior<_Sndrs, _Env...>()...);
|
||||
}
|
||||
};
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __attrs_t
|
||||
{
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT when_all_t::__sndr_t<>::__attrs_t : __inln_attrs_t
|
||||
{};
|
||||
|
||||
template <class... _Sndrs>
|
||||
_CCCL_HOST_DEVICE_API constexpr auto when_all_t::operator()(_Sndrs... __sndrs) const
|
||||
{
|
||||
if constexpr (sizeof...(_Sndrs) == 0)
|
||||
{
|
||||
return __sndr_t{};
|
||||
}
|
||||
else
|
||||
{
|
||||
// If the incoming senders are non-dependent, we can check the completion
|
||||
// signatures of the composed sender immediately.
|
||||
if constexpr (((!dependent_sender<_Sndrs>) && ...))
|
||||
{
|
||||
__assert_valid_completion_signatures(get_completion_signatures<__sndr_t<_Sndrs...>>());
|
||||
}
|
||||
|
||||
// If the incoming senders all know their completion domain, we can check
|
||||
// that they all share a common domain.
|
||||
if constexpr ((__callable<get_completion_domain_t<set_value_t>, env_of_t<_Sndrs>> && ...))
|
||||
{
|
||||
static_assert(
|
||||
__is_instantiable_with<::cuda::std::common_type_t, __completion_domain_of_t<set_value_t, _Sndrs>...>,
|
||||
"when_all: all child senders must share a common domain; that is, they must "
|
||||
"all complete on execution contexts that are similar in their execution semantics.");
|
||||
}
|
||||
|
||||
return __sndr_t<_Sndrs...>{{{}, {}, static_cast<_Sndrs&&>(__sndrs)...}};
|
||||
}
|
||||
}
|
||||
|
||||
template <class... _Sndrs>
|
||||
inline constexpr int structured_binding_size<when_all_t::__sndr_t<_Sndrs...>> = sizeof...(_Sndrs) + 2;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT when_all_t when_all{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_WHEN_ALL
|
||||
@@ -1,155 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_WRITE_ATTRS
|
||||
#define __CUDAX_EXECUTION_WRITE_ATTRS
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/get_completion_signatures.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
//! @brief Sender adaptor that adds attributes to the child sender's attributes.
|
||||
struct write_attrs_t
|
||||
{
|
||||
template <class _Sndr, class _Attrs>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
template <class _Attrs, class _SndrAttrs>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __attrs_t : env<__env_ref_t<_Attrs const&>, __fwd_env_t<_SndrAttrs>>
|
||||
{};
|
||||
|
||||
template <class _Attrs>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t
|
||||
{
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto operator()(_Sndr __sndr) &&
|
||||
{
|
||||
return __sndr_t<_Sndr, _Attrs>{{}, static_cast<_Attrs&&>(__attrs_), static_cast<_Sndr&&>(__sndr)};
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto operator()(_Sndr __sndr) const&
|
||||
{
|
||||
return __sndr_t<_Sndr, _Attrs>{{}, __attrs_, static_cast<_Sndr&&>(__sndr)};
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend auto operator|(_Sndr __sndr, __closure_t __clsr)
|
||||
{
|
||||
return __sndr_t<_Sndr, _Attrs>{{}, static_cast<_Attrs&&>(__clsr.__attrs_), static_cast<_Sndr&&>(__sndr)};
|
||||
}
|
||||
|
||||
_Attrs __attrs_;
|
||||
};
|
||||
|
||||
//! @brief Applies the given attributes to the sender and returns a new sender with the
|
||||
//! attributes attached.
|
||||
//!
|
||||
//! @tparam _Sndr The type of the sender.
|
||||
//! @tparam _Attrs The type of the attributes to be attached.
|
||||
//! @param __sndr The sender to which the attributes will be applied.
|
||||
//! @param __attrs The attributes to attach to the sender.
|
||||
//! @return A new sender type with the specified attributes attached.
|
||||
//!
|
||||
//! @note This function does not modify the original sender or attributes, but returns a new composed sender.
|
||||
//!
|
||||
//! **Example:**
|
||||
//! @rst
|
||||
//! .. code-block:: c++
|
||||
//!
|
||||
//! auto sndr = execution::write_attrs(execution::just(),
|
||||
// execution::prop{execution::get_domain, MyDomain{}});
|
||||
//! auto domain = execution::get_domain(execution::get_env(sndr));
|
||||
//! static_assert(std::is_same_v<decltype(domain), MyDomain>);
|
||||
//!
|
||||
//! @endrst
|
||||
template <class _Sndr, class _Attrs>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto operator()(_Sndr __sndr, _Attrs __attrs) const
|
||||
{
|
||||
return __sndr_t<_Sndr, _Attrs>{{}, static_cast<_Attrs&&>(__attrs), static_cast<_Sndr&&>(__sndr)};
|
||||
}
|
||||
|
||||
//! @brief Create a sender adaptor closure object that, when combined with a sender,
|
||||
//! will apply the specified attributes to that sender.
|
||||
//!
|
||||
//! @tparam _Attrs The type of the attribute object to be forwarded.
|
||||
//! @param __attrs The attribute object to be forwarded to the closure.
|
||||
//! @return An instance of `__closure_t<_Attrs>` constructed from the forwarded attributes.
|
||||
//!
|
||||
//! **Example:**
|
||||
//! @rst
|
||||
//! .. code-block:: c++
|
||||
//!
|
||||
//! auto sndr = execution::just()
|
||||
// | execution::write_attrs(execution::prop{execution::get_domain, MyDomain{}});
|
||||
//! auto domain = execution::get_domain(execution::get_env(sndr));
|
||||
//! static_assert(std::is_same_v<decltype(domain), MyDomain>);
|
||||
//!
|
||||
//! @endrst
|
||||
template <class _Attrs>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API auto operator()(_Attrs __attrs) const
|
||||
{
|
||||
return __closure_t<_Attrs>{static_cast<_Attrs&&>(__attrs)};
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Sndr, class _Attrs>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT write_attrs_t::__sndr_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class _Self, class... _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
return execution::get_child_completion_signatures<_Self, _Sndr, _Env...>();
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr _rcvr) && -> connect_result_t<_Sndr, _Rcvr>
|
||||
{
|
||||
return execution::connect(static_cast<_Sndr&&>(__sndr_), static_cast<_Rcvr&&>(_rcvr));
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr _rcvr) const& -> connect_result_t<_Sndr, _Rcvr>
|
||||
{
|
||||
return execution::connect(__sndr_, static_cast<_Rcvr&&>(_rcvr));
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __attrs_t<_Attrs, env_of_t<_Sndr>>
|
||||
{
|
||||
return {{__env_ref(__attrs_), __fwd_env(execution::get_env(__sndr_))}};
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ write_attrs_t __tag_;
|
||||
_Attrs __attrs_;
|
||||
_Sndr __sndr_;
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT write_attrs_t write_attrs{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_WRITE_ATTRS
|
||||
@@ -1,227 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDAX_EXECUTION_WRITE_ENV
|
||||
#define __CUDAX_EXECUTION_WRITE_ENV
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__utility/immovable.h>
|
||||
|
||||
#include <cuda/experimental/__detail/utility.cuh>
|
||||
#include <cuda/experimental/__execution/completion_signatures.cuh>
|
||||
#include <cuda/experimental/__execution/cpos.cuh>
|
||||
#include <cuda/experimental/__execution/env.cuh>
|
||||
#include <cuda/experimental/__execution/exception.cuh>
|
||||
#include <cuda/experimental/__execution/queries.cuh>
|
||||
#include <cuda/experimental/__execution/rcvr_ref.cuh>
|
||||
#include <cuda/experimental/__execution/rcvr_with_env.cuh>
|
||||
#include <cuda/experimental/__execution/transform_sender.cuh>
|
||||
#include <cuda/experimental/__execution/utility.cuh>
|
||||
#include <cuda/experimental/__execution/visit.cuh>
|
||||
|
||||
#include <cuda/experimental/__execution/prologue.cuh>
|
||||
|
||||
namespace cuda::experimental::execution
|
||||
{
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT write_env_t
|
||||
{
|
||||
_CUDAX_SEMI_PRIVATE :
|
||||
template <class _Rcvr, class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __state_t
|
||||
{
|
||||
_Rcvr __rcvr_;
|
||||
_Env __env_;
|
||||
};
|
||||
|
||||
template <class _Env, class... _RcvrEnv>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __env_ : env<__env_ref_t<_Env const&>, __fwd_env_t<_RcvrEnv>...>
|
||||
{
|
||||
using __base_t = env<__env_ref_t<_Env const&>, __fwd_env_t<_RcvrEnv>...>;
|
||||
|
||||
_CCCL_HOST_DEVICE_API explicit constexpr __env_(_Env const& env, _RcvrEnv&&... __rcvr_env) noexcept
|
||||
: __base_t{__env_ref(env), __fwd_env(static_cast<_RcvrEnv&&>(__rcvr_env))...}
|
||||
{}
|
||||
|
||||
using __base_t::query;
|
||||
|
||||
// If _Env has a value for the get_scheduler_t query, then make sure we are not
|
||||
// delegating the get_domain_t query to the receiver's environment.
|
||||
_CCCL_TEMPLATE(class _Env2 = _Env)
|
||||
_CCCL_REQUIRES((!__queryable_with<_Env2, get_domain_t>) )
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto query(get_domain_t) const noexcept
|
||||
-> __scheduler_domain_t<__scheduler_of_t<_Env2>, __fwd_env_t<_RcvrEnv>...>
|
||||
{
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Env, class... _RcvrEnv>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto
|
||||
__mk_env(const _Env& __env, _RcvrEnv&&... __rcvr_env) noexcept
|
||||
{
|
||||
return __env_{__env, static_cast<_RcvrEnv&&>(__rcvr_env)...};
|
||||
}
|
||||
|
||||
template <class _Env, class... _RcvrEnv>
|
||||
using __env_t = decltype(__mk_env(::cuda::std::declval<_Env>(), ::cuda::std::declval<_RcvrEnv>()...));
|
||||
|
||||
template <class _Rcvr, class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __rcvr_t
|
||||
{
|
||||
using receiver_concept = receiver_t;
|
||||
|
||||
template <class... _Ts>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_value(_Ts&&... __ts) noexcept
|
||||
{
|
||||
execution::set_value(static_cast<_Rcvr&&>(__state_->__rcvr_), static_cast<_Ts&&>(__ts)...);
|
||||
}
|
||||
|
||||
template <class _Error>
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_error(_Error&& __err) noexcept
|
||||
{
|
||||
execution::set_error(static_cast<_Rcvr&&>(__state_->__rcvr_), static_cast<_Error&&>(__err));
|
||||
}
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void set_stopped() noexcept
|
||||
{
|
||||
execution::set_stopped(static_cast<_Rcvr&&>(__state_->__rcvr_));
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __env_t<_Env, env_of_t<_Rcvr>>
|
||||
{
|
||||
return __mk_env(__state_->__env_, execution::get_env(__state_->__rcvr_));
|
||||
}
|
||||
|
||||
__state_t<_Rcvr, _Env>* __state_;
|
||||
};
|
||||
|
||||
template <class _Rcvr, class _Sndr, class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __opstate_t
|
||||
{
|
||||
using operation_state_concept = operation_state_t;
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr explicit __opstate_t(_Sndr&& __sndr, _Env __env, _Rcvr __rcvr)
|
||||
: __state_{static_cast<_Rcvr&&>(__rcvr), static_cast<_Env&&>(__env)}
|
||||
, __opstate_(execution::connect(static_cast<_Sndr&&>(__sndr), __rcvr_t<_Rcvr, _Env>{&__state_}))
|
||||
{}
|
||||
|
||||
_CCCL_IMMOVABLE(__opstate_t);
|
||||
|
||||
_CCCL_HOST_DEVICE_API constexpr void start() noexcept
|
||||
{
|
||||
execution::start(__opstate_);
|
||||
}
|
||||
|
||||
__state_t<_Rcvr, _Env> __state_;
|
||||
connect_result_t<_Sndr, __rcvr_t<_Rcvr, _Env>> __opstate_;
|
||||
};
|
||||
|
||||
public:
|
||||
template <class _Sndr, class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __sndr_t;
|
||||
|
||||
template <class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT __closure_t;
|
||||
|
||||
/// @brief Wraps one sender in another that modifies the execution
|
||||
/// environment by merging in the environment specified.
|
||||
template <class _Sndr, class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr __sndr, _Env __env) const
|
||||
{
|
||||
return __sndr_t<_Sndr, _Env>{{}, static_cast<_Env&&>(__env), static_cast<_Sndr&&>(__sndr)};
|
||||
}
|
||||
|
||||
/// @brief Returns a closure that can be used with the pipe operator
|
||||
/// to modify the execution environment.
|
||||
template <class _Env>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Env __env) const
|
||||
{
|
||||
return __closure_t<_Env>{static_cast<_Env&&>(__env)};
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Sndr, class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT write_env_t::__sndr_t
|
||||
{
|
||||
using sender_concept = sender_t;
|
||||
|
||||
template <class _Self, class... _RcvrEnv>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API static _CCCL_CONSTEVAL auto get_completion_signatures()
|
||||
{
|
||||
using _Child _CCCL_NODEBUG_ALIAS = ::cuda::std::__copy_cvref_t<_Self, _Sndr>;
|
||||
return execution::get_completion_signatures<_Child, __env_t<_Env, _RcvrEnv...>>();
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto connect(_Rcvr __rcvr) && -> __opstate_t<_Rcvr, _Sndr, _Env>
|
||||
{
|
||||
return __opstate_t<_Rcvr, _Sndr, _Env>{
|
||||
static_cast<_Sndr&&>(__sndr_), static_cast<_Env&&>(__env_), static_cast<_Rcvr&&>(__rcvr)};
|
||||
}
|
||||
|
||||
template <class _Rcvr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
|
||||
connect(_Rcvr __rcvr) const& -> __opstate_t<_Rcvr, const _Sndr&, _Env>
|
||||
{
|
||||
return __opstate_t<_Rcvr, const _Sndr&, _Env>{__sndr_, __env_, static_cast<_Rcvr&&>(__rcvr)};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_env() const noexcept -> __fwd_env_t<env_of_t<_Sndr>>
|
||||
{
|
||||
return __fwd_env(execution::get_env(__sndr_));
|
||||
}
|
||||
|
||||
/*_CCCL_NO_UNIQUE_ADDRESS*/ write_env_t __tag_;
|
||||
_Env __env_;
|
||||
_Sndr __sndr_;
|
||||
};
|
||||
|
||||
template <class _Env>
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT write_env_t::__closure_t
|
||||
{
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr __sndr) &&
|
||||
{
|
||||
return write_env_t()(static_cast<_Sndr&&>(__sndr), static_cast<_Env&&>(__env_));
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Sndr __sndr) const&
|
||||
{
|
||||
return write_env_t()(static_cast<_Sndr&&>(__sndr), __env_);
|
||||
}
|
||||
|
||||
template <class _Sndr>
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr auto operator|(_Sndr __sndr, __closure_t __self)
|
||||
{
|
||||
return write_env_t()(static_cast<_Sndr&&>(__sndr), static_cast<_Env&&>(__self.__env_));
|
||||
}
|
||||
|
||||
_Env __env_;
|
||||
};
|
||||
|
||||
template <class _Sndr, class _Env>
|
||||
inline constexpr int structured_binding_size<write_env_t::__sndr_t<_Sndr, _Env>> = 3;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT write_env_t write_env{};
|
||||
} // namespace cuda::experimental::execution
|
||||
|
||||
#include <cuda/experimental/__execution/epilogue.cuh>
|
||||
|
||||
#endif // __CUDAX_EXECUTION_WRITE_ENV
|
||||
Reference in New Issue
Block a user