[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
EngineX CI
@@ -0,0 +1,97 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_ADJACENT_DIFFERENCE_H
|
||||
#define _CUDA_STD___PSTL_ADJACENT_DIFFERENCE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__numeric/adjacent_difference.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/adjacent_difference.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _BinaryOp = ::cuda::std::minus<>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator adjacent_difference(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_BinaryOp __binary_op = {})
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__adjacent_difference,
|
||||
_Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::adjacent_difference");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::adjacent_difference requires at least one selected backend");
|
||||
return ::cuda::std::adjacent_difference(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_ADJACENT_DIFFERENCE_H
|
||||
@@ -0,0 +1,92 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_ADJACENT_FIND_H
|
||||
#define _CUDA_STD___PSTL_ADJACENT_FIND_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/zip_function.h>
|
||||
# include <cuda/__iterator/zip_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/adjacent_find.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/next.h>
|
||||
# include <cuda/std/__iterator/prev.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/find_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy,
|
||||
class _InputIterator,
|
||||
class _BinaryPredicate = ::cuda::std::equal_to<iter_value_t<_InputIterator>>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API _InputIterator adjacent_find(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _BinaryPredicate __pred = {})
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::adjacent_find");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
auto __zipped_ret = __dispatch(
|
||||
__policy,
|
||||
::cuda::zip_iterator{__first, ::cuda::std::next(__first)},
|
||||
::cuda::zip_iterator{::cuda::std::prev(__last), __last},
|
||||
::cuda::zip_function{::cuda::std::move(__pred)});
|
||||
return ::cuda::std::get<0>(__zipped_ret.__iterators());
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::adjacent_find requires at least one selected backend");
|
||||
return ::cuda::std::adjacent_find(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_ADJACENT_FIND_H
|
||||
87
cccl_upstream/libcudacxx/include/cuda/std/__pstl/all_of.h
Normal file
87
cccl_upstream/libcudacxx/include/cuda/std/__pstl/all_of.h
Normal file
@@ -0,0 +1,87 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_ALL_OF_H
|
||||
#define _CUDA_STD___PSTL_ALL_OF_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/all_of.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/not_fn.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_convertible.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/find_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _Iter, class _UnaryPred)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_Iter> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API bool
|
||||
all_of([[maybe_unused]] const _Policy& __policy, _Iter __first, _Iter __last, _UnaryPred __pred)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryPred, _Iter>,
|
||||
"cuda::std::all_of: UnaryOp must satisfy indirect_unary_predicate<Iter>");
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::all_of");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
auto __res =
|
||||
__dispatch(__policy, ::cuda::std::move(__first), __last, ::cuda::std::not_fn(::cuda::std::move(__pred)));
|
||||
return __res == __last;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::all_of requires at least one selected backend");
|
||||
return ::cuda::std::all_of(::cuda::std::move(__first), ::cuda::std::move(__last));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_ALL_OF_H
|
||||
87
cccl_upstream/libcudacxx/include/cuda/std/__pstl/any_of.h
Normal file
87
cccl_upstream/libcudacxx/include/cuda/std/__pstl/any_of.h
Normal file
@@ -0,0 +1,87 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_ANY_OF_H
|
||||
#define _CUDA_STD___PSTL_ANY_OF_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/any_of.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/not_fn.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_convertible.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/find_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _Iter, class _UnaryPred)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_Iter> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API bool
|
||||
any_of([[maybe_unused]] const _Policy& __policy, _Iter __first, _Iter __last, _UnaryPred __pred)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryPred, _Iter>,
|
||||
"cuda::std::any_of: UnaryOp must satisfy indirect_unary_predicate<Iter>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::any_of");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
auto __res = __dispatch(__policy, ::cuda::std::move(__first), __last, ::cuda::std::move(__pred));
|
||||
return __res != __last;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::any_of requires at least one selected backend");
|
||||
return ::cuda::std::any_of(::cuda::std::move(__first), ::cuda::std::move(__last));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_ANY_OF_H
|
||||
82
cccl_upstream/libcudacxx/include/cuda/std/__pstl/copy.h
Normal file
82
cccl_upstream/libcudacxx/include/cuda/std/__pstl/copy.h
Normal file
@@ -0,0 +1,82 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_COPY_H
|
||||
#define _CUDA_STD___PSTL_COPY_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/copy.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/copy_n.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator
|
||||
copy([[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _OutputIterator __result)
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__copy_n, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::copy");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
return __dispatch(__policy, ::cuda::std::move(__first), __count, ::cuda::std::move(__result));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::copy requires at least one selected backend");
|
||||
return ::cuda::std::copy(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__result));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_COPY_H
|
||||
93
cccl_upstream/libcudacxx/include/cuda/std/__pstl/copy_if.h
Normal file
93
cccl_upstream/libcudacxx/include/cuda/std/__pstl/copy_if.h
Normal file
@@ -0,0 +1,93 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_COPY_IF_H
|
||||
#define _CUDA_STD___PSTL_COPY_IF_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/copy_if.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/integral_constant.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/copy_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _UnaryPred)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator copy_if(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_UnaryPred __pred)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryPred, _InputIterator>,
|
||||
"cuda::std::copy_if: UnaryPred must satisfy indirect_unary_predicate<InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__copy_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::copy_if");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
return __dispatch(
|
||||
__policy, ::cuda::std::move(__first), __count, ::cuda::std::move(__result), ::cuda::std::move(__pred));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::copy_if requires at least one selected backend");
|
||||
return ::cuda::std::copy_if(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__result), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_COPY_IF_H
|
||||
83
cccl_upstream/libcudacxx/include/cuda/std/__pstl/copy_n.h
Normal file
83
cccl_upstream/libcudacxx/include/cuda/std/__pstl/copy_n.h
Normal file
@@ -0,0 +1,83 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_COPY_N_H
|
||||
#define _CUDA_STD___PSTL_COPY_N_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/copy_n.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/copy_n.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _Size)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator
|
||||
copy_n([[maybe_unused]] const _Policy& __policy, _InputIterator __first, _Size __count, _OutputIterator __result)
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__copy_n, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::copy_n");
|
||||
|
||||
if (__count == 0)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
return __dispatch(__policy,
|
||||
::cuda::std::move(__first),
|
||||
static_cast<iter_difference_t<_InputIterator>>(__count),
|
||||
::cuda::std::move(__result));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::copy_n requires at least one selected backend");
|
||||
return ::cuda::std::copy_n(::cuda::std::move(__first), __count, ::cuda::std::move(__result));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_COPY_N_H
|
||||
94
cccl_upstream/libcudacxx/include/cuda/std/__pstl/count.h
Normal file
94
cccl_upstream/libcudacxx/include/cuda/std/__pstl/count.h
Normal file
@@ -0,0 +1,94 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_COUNT_H
|
||||
#define _CUDA_STD___PSTL_COUNT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__functional/equal_to_value.h>
|
||||
# include <cuda/__iterator/transform_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/count.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_comparable.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/reduce.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _Tp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API iter_difference_t<_InputIterator>
|
||||
count([[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, const _Tp& __value)
|
||||
{
|
||||
static_assert(__is_cpp17_equality_comparable_v<iter_reference_t<_InputIterator>, _Tp>,
|
||||
"cuda::std::count: T must be equality comparable to Iter's value type.");
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__reduce, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::count");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return iter_difference_t<_InputIterator>{0};
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::transform_iterator{::cuda::std::move(__first), ::cuda::equal_to_value<_Tp>{__value}},
|
||||
__count,
|
||||
iter_difference_t<_InputIterator>{0},
|
||||
::cuda::std::plus<iter_difference_t<_InputIterator>>{});
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::count requires at least one selected backend");
|
||||
return ::cuda::std::count(::cuda::std::move(__first), ::cuda::std::move(__last), __value);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_COUNT_H
|
||||
94
cccl_upstream/libcudacxx/include/cuda/std/__pstl/count_if.h
Normal file
94
cccl_upstream/libcudacxx/include/cuda/std/__pstl/count_if.h
Normal file
@@ -0,0 +1,94 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_COUNT_IF_H
|
||||
#define _CUDA_STD___PSTL_COUNT_IF_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/transform_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/count_if.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_callable.h>
|
||||
# include <cuda/std/__type_traits/is_comparable.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/reduce.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _UnaryPredicate)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API iter_difference_t<_InputIterator> count_if(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _UnaryPredicate __pred)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryPredicate, _InputIterator>,
|
||||
"cuda::std::count_if: UnaryPred must satisfy indirect_unary_predicate<UnaryPred, Iter>");
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__reduce, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::count_if");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return iter_difference_t<_InputIterator>{0};
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::transform_iterator{::cuda::std::move(__first), ::cuda::std::move(__pred)},
|
||||
__count,
|
||||
iter_difference_t<_InputIterator>{0},
|
||||
::cuda::std::plus<iter_difference_t<_InputIterator>>{});
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::count_if requires at least one selected backend");
|
||||
return ::cuda::std::count_if(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_COUNT_IF_H
|
||||
@@ -0,0 +1,144 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_ADJACENT_DIFFERENCE_H
|
||||
#define _CUDA_STD___PSTL_CUDA_ADJACENT_DIFFERENCE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_adjacent_difference.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__numeric/adjacent_difference.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__adjacent_difference, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _OutputIterator, class _BinaryOp>
|
||||
[[nodiscard]] _CCCL_HOST_API static _OutputIterator __par_impl(
|
||||
const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_BinaryOp __binary_op)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
|
||||
// We pass the policy as an environment to DeviceAdjacentDifference
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceAdjacentDifference::SubtractLeftCopy,
|
||||
"__pstl_cuda_merge: kernel launch of cub::DeviceAdjacentDifference::SubtractLeftCopy failed",
|
||||
::cuda::std::move(__first),
|
||||
__result,
|
||||
__count,
|
||||
::cuda::std::move(__binary_op),
|
||||
__policy);
|
||||
|
||||
__stream.sync();
|
||||
|
||||
return __result + __count;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _BinaryOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _OutputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_BinaryOp __binary_op) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>
|
||||
&& ::cuda::std::__has_random_access_traversal<_OutputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "CUDA backend of cuda::std::merge requires random access iterators");
|
||||
return ::cuda::std::adjacent_difference(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_ADJACENT_DIFFERENCE_H
|
||||
173
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/copy_if.h
Normal file
173
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/copy_if.h
Normal file
@@ -0,0 +1,173 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_COPY_IF_H
|
||||
#define _CUDA_STD___PSTL_CUDA_COPY_IF_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_select.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/copy_if.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__iterator/next.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__copy_if, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _OutputIterator, class _UnaryPredicate>
|
||||
[[nodiscard]] _CCCL_HOST_API static _OutputIterator __par_impl(
|
||||
const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
iter_difference_t<_InputIterator> __count,
|
||||
_OutputIterator __result,
|
||||
_UnaryPredicate __pred)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
using _OffsetType = iter_difference_t<_InputIterator>;
|
||||
_OffsetType __ret;
|
||||
|
||||
// Determine temporary device storage requirements
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceSelect::If,
|
||||
"__pstl_cuda_select_if: determination of device storage for cub::DeviceSelect::If failed",
|
||||
static_cast<void*>(nullptr),
|
||||
__num_bytes,
|
||||
__first,
|
||||
__result,
|
||||
static_cast<_OffsetType*>(nullptr),
|
||||
__count,
|
||||
__pred,
|
||||
__policy);
|
||||
|
||||
{
|
||||
__temporary_storage<_OffsetType> __storage{__policy, __num_bytes, 1};
|
||||
|
||||
// Run the kernel
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceSelect::If,
|
||||
"__pstl_cuda_select_if: kernel launch of cub::DeviceSelect::If failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
::cuda::std::move(__first),
|
||||
__result,
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
__count,
|
||||
::cuda::std::move(__pred),
|
||||
__policy);
|
||||
|
||||
// Copy the result back from storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
::cudaMemcpyAsync,
|
||||
"__pstl_cuda_select_if: copy of result from device to host failed",
|
||||
::cuda::std::addressof(__ret),
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
sizeof(_OffsetType),
|
||||
::cudaMemcpyDefault,
|
||||
__stream.get());
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
return __result + static_cast<iter_difference_t<_OutputIterator>>(__ret);
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _UnaryPredicate)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_OutputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _OutputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
iter_difference_t<_InputIterator> __count,
|
||||
_OutputIterator __result,
|
||||
_UnaryPredicate __pred) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_OutputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(
|
||||
__policy, ::cuda::std::move(__first), __count, ::cuda::std::move(__result), ::cuda::std::move(__pred));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == ::cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_cuda_generate: CUDA backend of cuda::std::generate requires at least random access "
|
||||
"iterators");
|
||||
auto __last = ::cuda::std::next(__first, __count);
|
||||
return ::cuda::std::copy_if(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__result), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_COPY_IF_H
|
||||
144
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/copy_n.h
Normal file
144
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/copy_n.h
Normal file
@@ -0,0 +1,144 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_COPY_N_H
|
||||
#define _CUDA_STD___PSTL_CUDA_COPY_N_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/detail/choose_offset.cuh>
|
||||
# include <cub/device/device_transform.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/always_true_false.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/copy_if.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/identity.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__memory/pointer_traits.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_same.h>
|
||||
# include <cuda/std/__type_traits/is_trivially_copyable.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/cstdint>
|
||||
# include <cuda/std/tuple>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__copy_n, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _Size, class _OutputIterator, class _UnaryPred>
|
||||
[[nodiscard]] _CCCL_HOST_API static _OutputIterator __par_impl(
|
||||
const _Policy& __policy, _InputIterator __first, _Size __count, _OutputIterator __result, _UnaryPred __pred)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
// We pass the policy as an environment to DeviceTransform
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceTransform::TransformIf,
|
||||
"__pstl_cuda_copy_n: kernel launch of device_transform failed",
|
||||
tuple<_InputIterator>{::cuda::std::move(__first)},
|
||||
__result,
|
||||
__count,
|
||||
::cuda::std::move(__pred),
|
||||
identity{},
|
||||
__policy);
|
||||
|
||||
__stream.sync();
|
||||
|
||||
return __result + __count;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _UnaryPred = ::cuda::always_true)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _OutputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
iter_difference_t<_InputIterator> __count,
|
||||
_OutputIterator __result,
|
||||
_UnaryPred __pred = {}) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>
|
||||
&& ::cuda::std::__has_random_access_traversal<_OutputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(
|
||||
__policy, ::cuda::std::move(__first), __count, ::cuda::std::move(__result), ::cuda::std::move(__pred));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::copy_if requires at least random access iterators");
|
||||
auto __last = ::cuda::std::next(__first, __count);
|
||||
return ::cuda::std::copy_if(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__result), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_COPY_N_H
|
||||
@@ -0,0 +1,57 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_ENSURE_CURRENT_CONTEXT_H
|
||||
#define _CUDA_STD___PSTL_CUDA_ENSURE_CURRENT_CONTEXT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/__device/device_ref.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__runtime/ensure_current_context.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/std/__type_traits/is_callable.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
template <class _Policy>
|
||||
[[nodiscard]] _CCCL_HOST_API __ensure_current_context __pstl_ensure_current_ctx_for(const _Policy& __policy)
|
||||
{
|
||||
if constexpr (__is_callable_v<get_stream_t, const _Policy&>)
|
||||
{
|
||||
return __ensure_current_context{get_stream(__policy)};
|
||||
}
|
||||
else
|
||||
{
|
||||
int __curr_device{};
|
||||
_CCCL_TRY_CUDA_API(::cudaGetDevice, "Failed to get current device", &__curr_device);
|
||||
return __ensure_current_context{device_ref{__curr_device}};
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_ENSURE_CURRENT_CONTEXT_H
|
||||
@@ -0,0 +1,152 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_EXCLUSIVE_SCAN_H
|
||||
#define _CUDA_STD___PSTL_CUDA_EXCLUSIVE_SCAN_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_scan.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__iterator/tabulate_output_iterator.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__numeric/exclusive_scan.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__exclusive_scan, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp>
|
||||
[[nodiscard]] _CCCL_HOST_API static _OutputIterator __par_impl(
|
||||
const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
iter_difference_t<_InputIterator> __count,
|
||||
_OutputIterator __result,
|
||||
_BinaryOp __binary_op,
|
||||
_Tp __init)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
// We pass the policy as an environment to DeviceScan
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceScan::ExclusiveScan,
|
||||
"__pstl_cuda_exclusive_scan: kernel launch of cub::DeviceScan::ExclusiveScan failed",
|
||||
::cuda::std::move(__first),
|
||||
__result,
|
||||
::cuda::std::move(__binary_op),
|
||||
__init,
|
||||
__count,
|
||||
__policy);
|
||||
|
||||
__stream.sync();
|
||||
|
||||
return __result + iter_difference_t<_OutputIterator>(__count);
|
||||
}
|
||||
|
||||
template <class _Policy, class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp>
|
||||
[[nodiscard]] _CCCL_HOST_API _OutputIterator operator()(
|
||||
const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_Tp __init,
|
||||
_BinaryOp __binary_op) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>
|
||||
&& ::cuda::std::__has_random_access_traversal<_OutputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
return __par_impl(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
__count,
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op),
|
||||
__init);
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::exclusive_scan requires at least random access "
|
||||
"iterators");
|
||||
return ::cuda::std::exclusive_scan(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op),
|
||||
__init);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_EXCLUSIVE_SCAN_H
|
||||
163
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/find_if.h
Normal file
163
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/find_if.h
Normal file
@@ -0,0 +1,163 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_FIND_IF_H
|
||||
#define _CUDA_STD___PSTL_CUDA_FIND_IF_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_find.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/find_if.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__host_stdlib/new>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__memory/addressof.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__type_traits/remove_cvref.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__find_if, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _Iter, class _UnaryOp>
|
||||
[[nodiscard]] _CCCL_HOST_API static _Iter
|
||||
__par_impl([[maybe_unused]] const _Policy& __policy, _Iter __first, _Iter __last, _UnaryOp __pred)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
const auto __num_items = ::cuda::std::distance(__first, __last);
|
||||
using _OffsetType = CUB_NS_QUALIFIER::detail::choose_offset_t<remove_cvref_t<decltype(__num_items)>>;
|
||||
_OffsetType __ret;
|
||||
|
||||
// Determine temporary device storage requirements for find_if
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceFind::FindIf,
|
||||
"__pstl_cuda_find_if: determining temporary storage failed",
|
||||
static_cast<void*>(nullptr),
|
||||
__num_bytes,
|
||||
__first,
|
||||
static_cast<_OffsetType*>(nullptr),
|
||||
__pred,
|
||||
__num_items,
|
||||
__policy);
|
||||
|
||||
{
|
||||
__temporary_storage<_OffsetType> __storage{__policy, __num_bytes, 1};
|
||||
|
||||
// Run the find operation
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceFind::FindIf,
|
||||
"__pstl_cuda_find_if: cub::DeviceFind failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
::cuda::std::move(__first),
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
::cuda::std::move(__pred),
|
||||
__num_items,
|
||||
__policy);
|
||||
|
||||
// Copy the result back from storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
::cudaMemcpyAsync,
|
||||
"__pstl_cuda_find_if: copy of result from device to host failed",
|
||||
::cuda::std::addressof(__ret),
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
sizeof(_OffsetType),
|
||||
::cudaMemcpyDefault,
|
||||
__stream.get());
|
||||
}
|
||||
|
||||
// Need to sync before reading __ret
|
||||
__stream.sync();
|
||||
return __first + __ret;
|
||||
}
|
||||
|
||||
template <class _Policy, class _Iter, class _UnaryOp>
|
||||
[[nodiscard]] _CCCL_HOST_API _Iter
|
||||
operator()([[maybe_unused]] const _Policy& __policy, _Iter __first, _Iter __last, _UnaryOp __pred) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_Iter>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::find_if requires at least random access iterators");
|
||||
return ::cuda::std::find_if(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_FIND_IF_H
|
||||
@@ -0,0 +1,130 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_FOR_EACH_N_H
|
||||
#define _CUDA_STD___PSTL_CUDA_FOR_EACH_N_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_for.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/for_each_n.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__exception/terminate.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__host_stdlib/new>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/convert_to_integral.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <nv/target>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__for_each_n, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _Iter, class _Size, class _Fn>
|
||||
[[nodiscard]] _CCCL_HOST_API static _Iter
|
||||
__par_impl([[maybe_unused]] const _Policy& __policy, _Iter __first, _Size __orig_n, _Fn __func)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
const auto __count = ::cuda::std::__convert_to_integral(__orig_n);
|
||||
|
||||
// We pass the policy as an environment to DeviceFor
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceFor::ForEachN,
|
||||
"__pstl_dispatch: kernel launch failed",
|
||||
__first,
|
||||
__count,
|
||||
::cuda::std::move(__func),
|
||||
__policy);
|
||||
|
||||
__stream.sync();
|
||||
|
||||
return __first + static_cast<iter_difference_t<_Iter>>(__count);
|
||||
}
|
||||
|
||||
template <class _Policy, class _Iter, class _Size, class _Fn>
|
||||
[[nodiscard]] _CCCL_HOST_API _Iter operator()(const _Policy& __policy, _Iter __first, _Size __orig_n, _Fn __func) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_Iter>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(__policy, ::cuda::std::move(__first), __orig_n, ::cuda::std::move(__func));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == ::cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::for_each_n requires at least random access iterators");
|
||||
return ::cuda::std::for_each_n(::cuda::std::move(__first), __orig_n, ::cuda::std::move(__func));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_FOR_EACH_N_H
|
||||
@@ -0,0 +1,130 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_GENERATE_H
|
||||
#define _CUDA_STD___PSTL_CUDA_GENERATE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_transform.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/generate_n.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__host_stdlib/stdexcept>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/tuple>
|
||||
|
||||
# include <cuda_runtime.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__generate_n, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _OutputIterator, class _UnaryOp>
|
||||
[[nodiscard]] _CCCL_HOST_API static _OutputIterator
|
||||
__par_impl(const _Policy& __policy, _OutputIterator __result, const int64_t __count, _UnaryOp __func)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
// We pass the policy as an environment to DeviceTransform
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceTransform::Generate,
|
||||
"__pstl_cuda_generate: call to cub device_transform::Generate failed",
|
||||
__result,
|
||||
__count,
|
||||
::cuda::std::move(__func),
|
||||
__policy);
|
||||
|
||||
__stream.sync();
|
||||
|
||||
return __result + __count;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _OutputIterator, class _Size, class _UnaryOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_OutputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _OutputIterator
|
||||
operator()([[maybe_unused]] const _Policy& __policy, _OutputIterator __result, _Size __count, _UnaryOp __func) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_OutputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(__policy, ::cuda::std::move(__result), __count, ::cuda::std::move(__func));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_cuda_generate: CUDA backend of cuda::std::generate requires at least random access "
|
||||
"iterators");
|
||||
return ::cuda::std::generate_n(::cuda::std::move(__result), __count, ::cuda::std::move(__func));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_GENERATE_H
|
||||
@@ -0,0 +1,222 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_INCLUSIVE_SCAN_H
|
||||
#define _CUDA_STD___PSTL_CUDA_INCLUSIVE_SCAN_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_scan.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__iterator/tabulate_output_iterator.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__numeric/inclusive_scan.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__inclusive_scan, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp>
|
||||
[[nodiscard]] _CCCL_HOST_API static _OutputIterator __par_impl(
|
||||
const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
iter_difference_t<_InputIterator> __count,
|
||||
_OutputIterator __result,
|
||||
_BinaryOp __binary_op,
|
||||
_Tp __init)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
// We pass the policy as an environment to DeviceScan
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceScan::InclusiveScanInit,
|
||||
"__pstl_cuda_exclusive_scan: kernel launch of cub::DeviceScan::InclusiveScanInit failed",
|
||||
::cuda::std::move(__first),
|
||||
__result,
|
||||
::cuda::std::move(__binary_op),
|
||||
__init,
|
||||
__count,
|
||||
__policy);
|
||||
|
||||
__stream.sync();
|
||||
|
||||
return __result + iter_difference_t<_OutputIterator>(__count);
|
||||
}
|
||||
|
||||
template <class _Policy, class _InputIterator, class _OutputIterator, class _BinaryOp>
|
||||
[[nodiscard]] _CCCL_HOST_API static _OutputIterator __par_impl(
|
||||
const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
iter_difference_t<_InputIterator> __count,
|
||||
_OutputIterator __result,
|
||||
_BinaryOp __binary_op)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
_OutputIterator __ret = __result + iter_difference_t<_OutputIterator>(__count);
|
||||
|
||||
// We pass the policy as an environment to DeviceScan
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceScan::InclusiveScan,
|
||||
"__pstl_cuda_exclusive_scan: kernel launch of cub::DeviceScan::InclusiveScan failed",
|
||||
::cuda::std::move(__first),
|
||||
__result,
|
||||
::cuda::std::move(__binary_op),
|
||||
__count,
|
||||
__policy);
|
||||
|
||||
__stream.sync();
|
||||
return __ret;
|
||||
}
|
||||
|
||||
template <class _Policy, class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp>
|
||||
[[nodiscard]] _CCCL_HOST_API _OutputIterator operator()(
|
||||
const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_BinaryOp __binary_op,
|
||||
_Tp __init) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>
|
||||
&& ::cuda::std::__has_random_access_traversal<_OutputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
return __par_impl(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
__count,
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op),
|
||||
__init);
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::inclusive_scan requires at least random access "
|
||||
"iterators");
|
||||
return ::cuda::std::inclusive_scan(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op),
|
||||
__init);
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Policy, class _InputIterator, class _OutputIterator, class _BinaryOp>
|
||||
[[nodiscard]] _CCCL_HOST_API _OutputIterator operator()(
|
||||
const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_BinaryOp __binary_op) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>
|
||||
&& ::cuda::std::__has_random_access_traversal<_OutputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
return __par_impl(
|
||||
__policy, ::cuda::std::move(__first), __count, ::cuda::std::move(__result), ::cuda::std::move(__binary_op));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::inclusive_scan requires at least random access "
|
||||
"iterators");
|
||||
return ::cuda::std::inclusive_scan(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_INCLUSIVE_SCAN_H
|
||||
@@ -0,0 +1,162 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_MAX_ELEMENT_H
|
||||
#define _CUDA_STD___PSTL_CUDA_MAX_ELEMENT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_reduce.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__iterator/discard_iterator.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/max_element.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__memory/addressof.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/tuple>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__max_element, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _BinaryPred>
|
||||
[[nodiscard]] _CCCL_HOST_API static _InputIterator __par_impl(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _BinaryPred __pred)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
size_t __ret = 0ull;
|
||||
const auto __count = static_cast<int64_t>(::cuda::std::distance(__first, __last));
|
||||
|
||||
// Determine temporary device storage requirements for max_element
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceReduce::ArgMax,
|
||||
"__pstl_cuda_max_element: determination of device storage for cub::DeviceReduce::ArgMax failed",
|
||||
static_cast<void*>(nullptr),
|
||||
__num_bytes,
|
||||
__first,
|
||||
::cuda::discard_iterator{},
|
||||
static_cast<size_t*>(nullptr),
|
||||
__count,
|
||||
__pred,
|
||||
__policy);
|
||||
|
||||
{
|
||||
__temporary_storage<size_t> __storage{__policy, __num_bytes, 1};
|
||||
|
||||
// Run the reduction
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceReduce::ArgMax,
|
||||
"__pstl_cuda_max_element: kernel launch of cub::DeviceReduce::ArgMax failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
__first,
|
||||
::cuda::discard_iterator{},
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
__count,
|
||||
::cuda::std::move(__pred),
|
||||
__policy);
|
||||
|
||||
// Copy the result back from storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
::cudaMemcpyAsync,
|
||||
"__pstl_cuda_max_element: copy of result from device to host failed",
|
||||
::cuda::std::addressof(__ret),
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
sizeof(size_t),
|
||||
::cudaMemcpyDefault,
|
||||
__stream.get());
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
return __first + static_cast<iter_difference_t<_InputIterator>>(__ret);
|
||||
}
|
||||
|
||||
template <class _Policy, class _InputIterator, class _BinaryPred>
|
||||
[[nodiscard]] _CCCL_HOST_API _InputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _BinaryPred __pred) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::max_element requires at least random access "
|
||||
"iterators");
|
||||
return ::cuda::std::max_element(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_MAX_ELEMENT_H
|
||||
161
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/merge.h
Normal file
161
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/merge.h
Normal file
@@ -0,0 +1,161 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_MERGE_H
|
||||
#define _CUDA_STD___PSTL_CUDA_MERGE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_merge.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/merge.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__iterator/next.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__merge, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator1, class _InputIterator2, class _OutputIterator, class _Compare>
|
||||
[[nodiscard]] _CCCL_HOST_API static _OutputIterator __par_impl(
|
||||
const _Policy& __policy,
|
||||
_InputIterator1 __first1,
|
||||
_InputIterator1 __last1,
|
||||
_InputIterator2 __first2,
|
||||
_InputIterator2 __last2,
|
||||
_OutputIterator __result,
|
||||
_Compare __comp)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
iter_difference_t<_InputIterator1> __count1 = ::cuda::std::distance(__first1, __last1);
|
||||
iter_difference_t<_InputIterator2> __count2 = ::cuda::std::distance(__first2, __last2);
|
||||
auto __ret = __result + static_cast<iter_difference_t<_OutputIterator>>(__count1)
|
||||
+ static_cast<iter_difference_t<_OutputIterator>>(__count2);
|
||||
|
||||
// We pass the policy as an environment to DeviceMerge
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceMerge::MergeKeys,
|
||||
"__pstl_cuda_merge: kernel launch of cub::DeviceMerge::MergeKeys failed",
|
||||
::cuda::std::move(__first1),
|
||||
__count1,
|
||||
::cuda::std::move(__first2),
|
||||
__count2,
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__comp),
|
||||
__policy);
|
||||
|
||||
__stream.sync();
|
||||
|
||||
return __ret;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator1, class _InputIterator2, class _OutputIterator, class _Compare)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_OutputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _OutputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator1 __first1,
|
||||
_InputIterator1 __last1,
|
||||
_InputIterator2 __first2,
|
||||
_InputIterator2 __last2,
|
||||
_OutputIterator __result,
|
||||
_Compare __comp) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator1>
|
||||
&& ::cuda::std::__has_random_access_traversal<_InputIterator2>
|
||||
&& ::cuda::std::__has_random_access_traversal<_OutputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(
|
||||
__policy,
|
||||
::cuda::std::move(__first1),
|
||||
::cuda::std::move(__last1),
|
||||
::cuda::std::move(__first2),
|
||||
::cuda::std::move(__last2),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__comp));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "CUDA backend of cuda::std::merge requires random access iterators");
|
||||
return ::cuda::std::merge(
|
||||
::cuda::std::move(__first1),
|
||||
::cuda::std::move(__last1),
|
||||
::cuda::std::move(__first2),
|
||||
::cuda::std::move(__last2),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__comp));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_MERGE_H
|
||||
@@ -0,0 +1,162 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_MIN_ELEMENT_H
|
||||
#define _CUDA_STD___PSTL_CUDA_MIN_ELEMENT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_reduce.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__iterator/discard_iterator.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/min_element.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__memory/addressof.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/tuple>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__min_element, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _BinaryPred>
|
||||
[[nodiscard]] _CCCL_HOST_API static _InputIterator __par_impl(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _BinaryPred __pred)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
size_t __ret = 0ull;
|
||||
const auto __count = static_cast<int64_t>(::cuda::std::distance(__first, __last));
|
||||
|
||||
// Determine temporary device storage requirements for min_element
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceReduce::ArgMin,
|
||||
"__pstl_cuda_min_element: determination of device storage for cub::DeviceReduce::ArgMin failed",
|
||||
static_cast<void*>(nullptr),
|
||||
__num_bytes,
|
||||
__first,
|
||||
::cuda::discard_iterator{},
|
||||
static_cast<size_t*>(nullptr),
|
||||
__count,
|
||||
__pred,
|
||||
__policy);
|
||||
|
||||
{
|
||||
__temporary_storage<size_t> __storage{__policy, __num_bytes, 1};
|
||||
|
||||
// Run the reduction
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceReduce::ArgMin,
|
||||
"__pstl_cuda_min_element: kernel launch of cub::DeviceReduce::ArgMin failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
__first,
|
||||
::cuda::discard_iterator{},
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
__count,
|
||||
::cuda::std::move(__pred),
|
||||
__policy);
|
||||
|
||||
// Copy the result back from storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
::cudaMemcpyAsync,
|
||||
"__pstl_cuda_min_element: copy of result from device to host failed",
|
||||
::cuda::std::addressof(__ret),
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
sizeof(size_t),
|
||||
::cudaMemcpyDefault,
|
||||
__stream.get());
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
return __first + static_cast<iter_difference_t<_InputIterator>>(__ret);
|
||||
}
|
||||
|
||||
template <class _Policy, class _InputIterator, class _BinaryPred>
|
||||
[[nodiscard]] _CCCL_HOST_API _InputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _BinaryPred __pred) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::min_element requires at least random access "
|
||||
"iterators");
|
||||
return ::cuda::std::min_element(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_MIN_ELEMENT_H
|
||||
@@ -0,0 +1,174 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_PARTITION_H
|
||||
#define _CUDA_STD___PSTL_CUDA_PARTITION_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_partition.cuh>
|
||||
# include <cub/device/device_transform.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/always_true_false.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/partition.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/identity.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__partition, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _UnaryPred>
|
||||
[[nodiscard]] _CCCL_HOST_API static _InputIterator
|
||||
__par_impl(const _Policy& __policy, _InputIterator __first, _InputIterator __last, _UnaryPred __pred)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
using _OffsetType = size_t;
|
||||
using value_type = iter_value_t<_InputIterator>;
|
||||
|
||||
_OffsetType __num_selected;
|
||||
const auto __count = static_cast<_OffsetType>(::cuda::std::distance(__first, __last));
|
||||
|
||||
// Determine temporary device storage requirements for device_partition
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DevicePartition::If,
|
||||
"__pstl_cuda_partition: determination of device storage for cub::DevicePartition::If failed",
|
||||
static_cast<void*>(nullptr),
|
||||
__num_bytes,
|
||||
static_cast<value_type*>(nullptr),
|
||||
__first,
|
||||
static_cast<_OffsetType*>(nullptr),
|
||||
__count,
|
||||
__pred,
|
||||
__policy);
|
||||
|
||||
{
|
||||
__temporary_storage<_OffsetType, value_type> __storage{__policy, __num_bytes, 1, __count};
|
||||
|
||||
// Partition cannot run inplace, so we need to first copy the input into temporary storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceTransform::TransformIf,
|
||||
"__pstl_cuda_partition: kernel launch of cub::DeviceTransform::TransformIf failed",
|
||||
tuple<_InputIterator>{__first},
|
||||
__storage.template __get_ptr<1>(),
|
||||
__count,
|
||||
::cuda::always_true{},
|
||||
identity{},
|
||||
__policy);
|
||||
|
||||
// Run the kernel, the standard requires that the input and output range do not overlap
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DevicePartition::If,
|
||||
"__pstl_cuda_partition: kernel launch of cub::DevicePartition::If failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
__storage.template __get_raw_ptr<1>(),
|
||||
__first,
|
||||
__storage.template __get_ptr<0>(),
|
||||
__count,
|
||||
::cuda::std::move(__pred),
|
||||
__policy);
|
||||
|
||||
// Copy the result back from storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
::cudaMemcpyAsync,
|
||||
"__pstl_cuda_partition: copy of result from device to host failed",
|
||||
::cuda::std::addressof(__num_selected),
|
||||
__storage.template __get_ptr<0>(),
|
||||
sizeof(_OffsetType),
|
||||
::cudaMemcpyDefault,
|
||||
__stream.get());
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
return __first + static_cast<iter_difference_t<_InputIterator>>(__num_selected);
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _UnaryPred)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _InputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _UnaryPred __pred) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "CUDA backend of cuda::std::partition requires random access iterators");
|
||||
return ::cuda::std::partition(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_PARTITION_H
|
||||
@@ -0,0 +1,191 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_PARTITION_COPY_H
|
||||
#define _CUDA_STD___PSTL_CUDA_PARTITION_COPY_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_partition.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/partition_copy.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/__utility/pair.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__partition_copy, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPred>
|
||||
[[nodiscard]] _CCCL_HOST_API static pair<_OutputIterator1, _OutputIterator2> __par_impl(
|
||||
const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator1 __result_true,
|
||||
_OutputIterator2 __result_false,
|
||||
_UnaryPred __pred)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
using _OffsetType = size_t;
|
||||
using __output_wrapper_t =
|
||||
CUB_NS_QUALIFIER::detail::select::partition_distinct_output_t<_OutputIterator1, _OutputIterator2>;
|
||||
__output_wrapper_t __result{__result_true, __result_false};
|
||||
|
||||
_OffsetType __num_selected;
|
||||
const auto __count = static_cast<_OffsetType>(::cuda::std::distance(__first, __last));
|
||||
|
||||
// Determine temporary device storage requirements for device_partition
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DevicePartition::If,
|
||||
"__pstl_cuda_partition_copy: determination of device storage for cub::DevicePartition::If failed",
|
||||
static_cast<void*>(nullptr),
|
||||
__num_bytes,
|
||||
__first,
|
||||
__result,
|
||||
static_cast<_OffsetType*>(nullptr),
|
||||
__count,
|
||||
__pred,
|
||||
__policy);
|
||||
|
||||
{
|
||||
__temporary_storage<_OffsetType> __storage{__policy, __num_bytes, 1};
|
||||
|
||||
// Run the kernel, the standard requires that the input and output range do not overlap
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DevicePartition::If,
|
||||
"__pstl_cuda_partition_copy: kernel launch of cub::DevicePartition::If failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__result),
|
||||
__storage.template __get_ptr<0>(),
|
||||
__count,
|
||||
::cuda::std::move(__pred),
|
||||
__policy);
|
||||
|
||||
// Copy the result back from storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
::cudaMemcpyAsync,
|
||||
"__pstl_cuda_partition_copy: copy of result from device to host failed",
|
||||
::cuda::std::addressof(__num_selected),
|
||||
__storage.template __get_ptr<0>(),
|
||||
sizeof(_OffsetType),
|
||||
::cudaMemcpyDefault,
|
||||
__stream.get());
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
const auto __num_not_selected = __count - static_cast<iter_difference_t<_InputIterator>>(__num_selected);
|
||||
return pair{__result_true + static_cast<iter_difference_t<_OutputIterator1>>(__num_selected),
|
||||
__result_false + static_cast<iter_difference_t<_OutputIterator2>>(__num_not_selected)};
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPred)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator1> _CCCL_AND
|
||||
__has_forward_traversal<_OutputIterator2>)
|
||||
[[nodiscard]] _CCCL_HOST_API pair<_OutputIterator1, _OutputIterator2> operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator1 __result_true,
|
||||
_OutputIterator2 __result_false,
|
||||
_UnaryPred __pred) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>
|
||||
&& ::cuda::std::__has_random_access_traversal<_OutputIterator1>
|
||||
&& ::cuda::std::__has_random_access_traversal<_OutputIterator2>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result_true),
|
||||
::cuda::std::move(__result_false),
|
||||
::cuda::std::move(__pred));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"CUDA backend of cuda::std::partition_copy requires random access iterators");
|
||||
return ::cuda::std::partition_copy(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result_true),
|
||||
::cuda::std::move(__result_false),
|
||||
::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_PARTITION_COPY_H
|
||||
180
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/reduce.h
Normal file
180
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/reduce.h
Normal file
@@ -0,0 +1,180 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_REDUCE_H
|
||||
#define _CUDA_STD___PSTL_CUDA_REDUCE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_reduce.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__iterator/tabulate_output_iterator.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/invoke.h>
|
||||
# include <cuda/std/__host_stdlib/new>
|
||||
# include <cuda/std/__host_stdlib/stdexcept>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__iterator/next.h>
|
||||
# include <cuda/std/__memory/addressof.h>
|
||||
# include <cuda/std/__memory/construct_at.h>
|
||||
# include <cuda/std/__numeric/reduce.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_constructible.h>
|
||||
# include <cuda/std/__utility/forward.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__reduce, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _Iter, class _Size, class _Tp, class _BinaryOp>
|
||||
[[nodiscard]] _CCCL_HOST_API static _Tp
|
||||
__par_impl([[maybe_unused]] const _Policy& __policy, _Iter __first, _Size __count, _Tp __init, _BinaryOp __func)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
_Tp __ret;
|
||||
|
||||
// We need to know the accumulator type to determine whether we need construct_at for the return value
|
||||
using _AccumT = __accumulator_t<_BinaryOp, iter_reference_t<_Iter>, _Tp>;
|
||||
|
||||
// Determine temporary device storage requirements for reduce
|
||||
void* __temp_storage = nullptr;
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceReduce::Reduce,
|
||||
"__pstl_cuda_reduce: determination of device storage for cub::DeviceReduce::Reduce failed",
|
||||
__temp_storage,
|
||||
__num_bytes,
|
||||
__first,
|
||||
static_cast<_Tp*>(nullptr),
|
||||
__count,
|
||||
__func,
|
||||
__init);
|
||||
|
||||
{
|
||||
__temporary_storage<_Tp> __storage{__policy, __num_bytes, 1};
|
||||
|
||||
// Run the reduction
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceReduce::Reduce,
|
||||
"__pstl_cuda_reduce: kernel launch of cub::DeviceReduce::Reduce failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
::cuda::std::move(__first),
|
||||
__storage.template __get_ptr<0, _AccumT>(),
|
||||
__count,
|
||||
::cuda::std::move(__func),
|
||||
::cuda::std::move(__init),
|
||||
__stream.get());
|
||||
|
||||
// Copy the result back from storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
::cudaMemcpyAsync,
|
||||
"__pstl_cuda_reduce: copy of result from device to host failed",
|
||||
::cuda::std::addressof(__ret),
|
||||
__storage.template __get_ptr<0>(),
|
||||
sizeof(_Tp),
|
||||
::cudaMemcpyDefault,
|
||||
__stream.get());
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
return __ret;
|
||||
}
|
||||
|
||||
template <class _Policy, class _Iter, class _Size, class _Tp, class _BinaryOp>
|
||||
[[nodiscard]] _CCCL_HOST_API _Tp
|
||||
operator()([[maybe_unused]] const _Policy& __policy, _Iter __first, _Size __count, _Tp __init, _BinaryOp __func) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_Iter>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(
|
||||
__policy, ::cuda::std::move(__first), __count, ::cuda::std::move(__init), ::cuda::std::move(__func));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::reduce requires at least random access iterators");
|
||||
return ::cuda::std::reduce(
|
||||
__first, ::cuda::std::next(__first, __count), ::cuda::std::move(__init), ::cuda::std::move(__func));
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Policy, class _Iter, class _Tp, class _BinaryOp>
|
||||
[[nodiscard]] _CCCL_HOST_API _Tp
|
||||
operator()([[maybe_unused]] const _Policy& __policy, _Iter __first, _Iter __last, _Tp __init, _BinaryOp __func) const
|
||||
{
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
return (*this)(__policy, ::cuda::std::move(__first), __count, ::cuda::std::move(__init), ::cuda::std::move(__func));
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_REDUCE_H
|
||||
@@ -0,0 +1,162 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_REMOVE_IF_H
|
||||
#define _CUDA_STD___PSTL_CUDA_REMOVE_IF_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_select.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/remove_if.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__iterator/next.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__remove_if, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _UnaryPredicate>
|
||||
[[nodiscard]] _CCCL_HOST_API static _InputIterator __par_impl(
|
||||
const _Policy& __policy, _InputIterator __first, iter_difference_t<_InputIterator> __count, _UnaryPredicate __pred)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
using _OffsetType = iter_difference_t<_InputIterator>;
|
||||
_OffsetType __ret;
|
||||
|
||||
// Determine temporary device storage requirements
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceSelect::If,
|
||||
"__pstl_cuda_select_if: determination of device storage for cub::DeviceSelect::If failed",
|
||||
static_cast<void*>(nullptr),
|
||||
__num_bytes,
|
||||
__first,
|
||||
static_cast<_OffsetType*>(nullptr),
|
||||
__count,
|
||||
__pred,
|
||||
__policy);
|
||||
|
||||
{
|
||||
__temporary_storage<_OffsetType> __storage{__policy, __num_bytes, 1};
|
||||
|
||||
// Run the kernel
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceSelect::If,
|
||||
"__pstl_cuda_select_if: kernel launch of cub::DeviceSelect::If failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
::cuda::std::move(__first),
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
__count,
|
||||
::cuda::std::move(__pred),
|
||||
__policy);
|
||||
|
||||
// Copy the result back from storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
::cudaMemcpyAsync,
|
||||
"__pstl_cuda_select_if: copy of result from device to host failed",
|
||||
::cuda::std::addressof(__ret),
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
sizeof(_OffsetType),
|
||||
::cudaMemcpyDefault,
|
||||
__stream.get());
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
return __first + __ret;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _UnaryPredicate)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _InputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
iter_difference_t<_InputIterator> __count,
|
||||
_UnaryPredicate __pred) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(__policy, ::cuda::std::move(__first), __count, ::cuda::std::move(__pred));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == ::cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_cuda_generate: CUDA backend of cuda::std::generate requires at least random access "
|
||||
"iterators");
|
||||
auto __last = ::cuda::std::next(__first, __count);
|
||||
return ::cuda::std::remove_if(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_REMOVE_IF_H
|
||||
186
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/rotate.h
Normal file
186
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/rotate.h
Normal file
@@ -0,0 +1,186 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_ROTATE_H
|
||||
#define _CUDA_STD___PSTL_CUDA_ROTATE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_partition.cuh>
|
||||
# include <cub/device/device_transform.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/always_true_false.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__iterator/counting_iterator.h>
|
||||
# include <cuda/__iterator/transform_iterator.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/rotate.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/identity.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
struct __rotate_fn
|
||||
{
|
||||
size_t __middle_;
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr bool operator()(size_t __index) const noexcept
|
||||
{
|
||||
return __index >= __middle_;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__rotate, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator>
|
||||
[[nodiscard]] _CCCL_HOST_API static _InputIterator
|
||||
__par_impl(const _Policy& __policy, _InputIterator __first, _InputIterator __middle, _InputIterator __last)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
using _OffsetType = size_t;
|
||||
using value_type = iter_value_t<_InputIterator>;
|
||||
|
||||
const auto __count = static_cast<_OffsetType>(::cuda::std::distance(__first, __last));
|
||||
const auto __count1 = static_cast<_OffsetType>(::cuda::std::distance(__first, __middle));
|
||||
auto __result = __first + static_cast<iter_difference_t<_InputIterator>>(__count - __count1);
|
||||
|
||||
// Knowing the sizes of the partitions, we can directly write into them
|
||||
using __output_wrapper_t =
|
||||
CUB_NS_QUALIFIER::detail::select::partition_distinct_output_t<_InputIterator, _InputIterator>;
|
||||
__output_wrapper_t __output_wrapper{__first, __result};
|
||||
|
||||
// Determine temporary device storage requirements for cub::DevicePartition::Flagged
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DevicePartition::Flagged,
|
||||
"__pstl_cuda_rotate: determination of device storage for cub::DevicePartition::Flagged failed",
|
||||
static_cast<void*>(nullptr),
|
||||
__num_bytes,
|
||||
static_cast<value_type*>(nullptr),
|
||||
::cuda::transform_iterator{::cuda::counting_iterator<size_t>{0}, __rotate_fn{__count1}},
|
||||
__output_wrapper,
|
||||
static_cast<_OffsetType*>(nullptr),
|
||||
__count,
|
||||
__policy);
|
||||
|
||||
{
|
||||
// Allocate memory for result
|
||||
__temporary_storage<_OffsetType, value_type> __storage{__policy, __num_bytes, 1, __count};
|
||||
|
||||
// Partition cannot run inplace, so we need to first copy the input into temporary storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceTransform::TransformIf,
|
||||
"__pstl_cuda_rotate: kernel launch of cub::DeviceTransform::TransformIf failed",
|
||||
tuple<_InputIterator>{::cuda::std::move(__first)},
|
||||
__storage.template __get_ptr<1>(),
|
||||
__count,
|
||||
::cuda::always_true{},
|
||||
identity{},
|
||||
__policy);
|
||||
|
||||
// Run the kernel, we use the flagged kernel because we know the exact ordering we want
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DevicePartition::Flagged,
|
||||
"__pstl_cuda_rotate: kernel launch of cub::DevicePartition::Flagged failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
__storage.template __get_raw_ptr<1>(),
|
||||
::cuda::transform_iterator{::cuda::counting_iterator<size_t>{0}, __rotate_fn{__count1}},
|
||||
::cuda::std::move(__output_wrapper),
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
__count,
|
||||
__policy);
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
return __result;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _InputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __middle,
|
||||
_InputIterator __last) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(__policy, ::cuda::std::move(__first), ::cuda::std::move(__middle), ::cuda::std::move(__last));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "CUDA backend of cuda::std::rotate requires random access iterators");
|
||||
return ::cuda::std::rotate(::cuda::std::move(__first), ::cuda::std::move(__middle), ::cuda::std::move(__last));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_ROTATE_H
|
||||
@@ -0,0 +1,185 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_ROTATE_COPY_H
|
||||
#define _CUDA_STD___PSTL_CUDA_ROTATE_COPY_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_partition.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__iterator/counting_iterator.h>
|
||||
# include <cuda/__iterator/transform_iterator.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/rotate_copy.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/identity.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
struct __rotate_copy_fn
|
||||
{
|
||||
size_t __middle_;
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr bool operator()(size_t __index) const noexcept
|
||||
{
|
||||
return __index >= __middle_;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__rotate_copy, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _OutputIterator>
|
||||
[[nodiscard]] _CCCL_HOST_API static _OutputIterator __par_impl(
|
||||
const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __middle,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
using _OffsetType = size_t;
|
||||
|
||||
const auto __count = static_cast<_OffsetType>(::cuda::std::distance(__first, __last));
|
||||
const auto __count1 = static_cast<_OffsetType>(::cuda::std::distance(__first, __middle));
|
||||
|
||||
// Knowing the sizes of the partitions, we can directly write into them
|
||||
using __output_wrapper_t =
|
||||
CUB_NS_QUALIFIER::detail::select::partition_distinct_output_t<_OutputIterator, _OutputIterator>;
|
||||
__output_wrapper_t __output_wrapper{
|
||||
__result, __result + static_cast<iter_difference_t<_OutputIterator>>(__count - __count1)};
|
||||
|
||||
// Determine temporary device storage requirements for cub::DevicePartition::Flagged
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DevicePartition::Flagged,
|
||||
"__pstl_cuda_rotate_copy: determination of device storage for cub::DevicePartition::Flagged failed",
|
||||
static_cast<void*>(nullptr),
|
||||
__num_bytes,
|
||||
__first,
|
||||
::cuda::transform_iterator{::cuda::counting_iterator<size_t>{0}, __rotate_copy_fn{__count1}},
|
||||
__output_wrapper,
|
||||
static_cast<_OffsetType*>(nullptr),
|
||||
__count,
|
||||
__policy);
|
||||
|
||||
{
|
||||
// Allocate memory for result
|
||||
__temporary_storage<_OffsetType> __storage{__policy, __num_bytes, 1};
|
||||
|
||||
// Run the kernel, we use the flagged kernel because we know the exact ordering we want
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DevicePartition::Flagged,
|
||||
"__pstl_cuda_rotate_copy: kernel launch of cub::DevicePartition::Flagged failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::transform_iterator{::cuda::counting_iterator<size_t>{0}, __rotate_copy_fn{__count1}},
|
||||
::cuda::std::move(__output_wrapper),
|
||||
__storage.template __get_ptr<0>(),
|
||||
__count,
|
||||
__policy);
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
return __result + static_cast<iter_difference_t<_OutputIterator>>(__count);
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _OutputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __middle,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>
|
||||
&& ::cuda::std::__has_random_access_traversal<_OutputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__middle),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"CUDA backend of cuda::std::rotate_copy requires random access iterators");
|
||||
return ::cuda::std::rotate_copy(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__middle), ::cuda::std::move(__last), ::cuda::std::move(__result));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_ROTATE_COPY_H
|
||||
@@ -0,0 +1,168 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_SHIFT_LEFT_H
|
||||
#define _CUDA_STD___PSTL_CUDA_SHIFT_LEFT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_select.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__iterator/counting_iterator.h>
|
||||
# include <cuda/__iterator/transform_iterator.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/shift_left.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
struct __shift_left_predicate
|
||||
{
|
||||
size_t __start_;
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr bool operator()(size_t __index) const noexcept
|
||||
{
|
||||
return __index >= __start_;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__shift_left, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator>
|
||||
[[nodiscard]] _CCCL_HOST_API static _InputIterator __par_impl(
|
||||
const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
iter_difference_t<_InputIterator> __num_shifted)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
using _OffsetType = iter_difference_t<_InputIterator>;
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
const auto __result = __first + static_cast<_OffsetType>(__count - __num_shifted);
|
||||
auto __flag_iter = ::cuda::transform_iterator{
|
||||
::cuda::counting_iterator<size_t>{0}, __shift_left_predicate{static_cast<size_t>(__num_shifted)}};
|
||||
|
||||
// Determine temporary device storage requirements for DeviceSelect::Flagged
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceSelect::Flagged,
|
||||
"__pstl_cuda_shift_left: determination of device storage for cub::DeviceSelect::Flagged failed",
|
||||
static_cast<void*>(nullptr),
|
||||
__num_bytes,
|
||||
__first,
|
||||
__flag_iter,
|
||||
static_cast<_OffsetType*>(nullptr),
|
||||
__count,
|
||||
__policy);
|
||||
|
||||
{
|
||||
__temporary_storage<_OffsetType> __storage{__policy, __num_bytes, 1};
|
||||
|
||||
// Run the kernel
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceSelect::Flagged,
|
||||
"__pstl_cuda_shift_left: kernel launch of cub::DeviceSelect::Flagged failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__flag_iter),
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
__count,
|
||||
__policy);
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
return __result;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _InputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
iter_difference_t<_InputIterator> __num_shifted) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), __num_shifted);
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"CUDA backend of cuda::std::shift_left requires random access iterators");
|
||||
return ::cuda::std::shift_left(::cuda::std::move(__first), ::cuda::std::move(__last), __num_shifted);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_SHIFT_LEFT_H
|
||||
@@ -0,0 +1,197 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_SHIFT_RIGHT_H
|
||||
#define _CUDA_STD___PSTL_CUDA_SHIFT_RIGHT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_transform.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/shift_right.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/identity.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__memory/pointer_traits.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/cstdint>
|
||||
# include <cuda/std/tuple>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__shift_right, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator>
|
||||
[[nodiscard]] _CCCL_HOST_API static _InputIterator __par_impl(
|
||||
const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
iter_difference_t<_InputIterator> __num_shifted)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
using _OffsetType = iter_difference_t<_InputIterator>;
|
||||
using value_type = iter_value_t<_InputIterator>;
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
const auto __count_remaining = static_cast<_OffsetType>(__count - __num_shifted);
|
||||
const auto __result = __first + __num_shifted;
|
||||
|
||||
if (2 * __num_shifted > __count)
|
||||
{ // There is no overlap between the source and destination, so we can just copy
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceTransform::Transform,
|
||||
"__pstl_cuda_shift_right: first kernel launch of cub::DeviceTransform::Transform failed",
|
||||
tuple<_InputIterator>{__first},
|
||||
__result,
|
||||
__count_remaining,
|
||||
identity{},
|
||||
__stream.get());
|
||||
}
|
||||
else if (3 * __num_shifted > __count)
|
||||
{ // We do need two copies, but we can avoid temporary storage
|
||||
const auto __count_second_batch = static_cast<_OffsetType>(__count_remaining - __num_shifted);
|
||||
// The first batch is __num_shifted elements, starting at the end of the second batch
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceTransform::Transform,
|
||||
"__pstl_cuda_shift_right: first kernel launch of cub::DeviceTransform::Transform failed",
|
||||
tuple<_InputIterator>{__first + __count_second_batch},
|
||||
__result + __count_second_batch,
|
||||
__num_shifted,
|
||||
identity{},
|
||||
__stream.get());
|
||||
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceTransform::Transform,
|
||||
"__pstl_cuda_shift_right: second kernel launch of cub::DeviceTransform::Transform failed",
|
||||
tuple<_InputIterator>{__first},
|
||||
__result,
|
||||
__count_second_batch,
|
||||
identity{},
|
||||
__stream.get());
|
||||
}
|
||||
else
|
||||
{ // Need temporary storage
|
||||
size_t __num_bytes = 1;
|
||||
__temporary_storage<value_type> __storage{__policy, __num_bytes, static_cast<size_t>(__count - __num_shifted)};
|
||||
|
||||
// Run the kernel to copy to temporary storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceTransform::Transform,
|
||||
"__pstl_cuda_shift_right: first kernel launch of cub::DeviceTransform::Transform failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
tuple<_InputIterator>{::cuda::std::move(__first)},
|
||||
__storage.template __get_ptr<0>(),
|
||||
__count_remaining,
|
||||
identity{},
|
||||
__stream.get());
|
||||
|
||||
// Run the kernel to copy back from temporary storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceTransform::Transform,
|
||||
"__pstl_cuda_shift_right: second kernel launch of cub::DeviceTransform::Transform failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
tuple<value_type*>{__storage.template __get_ptr<0>()},
|
||||
__result,
|
||||
__count_remaining,
|
||||
identity{},
|
||||
__stream.get());
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
return __result;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _InputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
iter_difference_t<_InputIterator> __num_shifted) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), __num_shifted);
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::shift_right requires at least random access "
|
||||
"iterators");
|
||||
return ::cuda::std::shift_right(::cuda::std::move(__first), ::cuda::std::move(__last), __num_shifted);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_SHIFT_RIGHT_H
|
||||
218
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/sort.h
Normal file
218
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/sort.h
Normal file
@@ -0,0 +1,218 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_SORT_H
|
||||
#define _CUDA_STD___PSTL_CUDA_SORT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_merge_sort.cuh>
|
||||
# include <cub/device/device_radix_sort.cuh>
|
||||
# include <cub/device/device_transform.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__cmath/round_up.h>
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/always_true_false.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/sort.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_one_of.h>
|
||||
# include <cuda/std/__type_traits/remove_cvref.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__sort, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Tp>
|
||||
using _DeviceRadixSort =
|
||||
cudaError_t (*)(void*, size_t&, CUB_NS_QUALIFIER::DoubleBuffer<_Tp>&, size_t, int, int, cudaStream_t);
|
||||
|
||||
template <class _Tp, class _BinaryPredicate>
|
||||
[[nodiscard]] static _CCCL_CONSTEVAL _DeviceRadixSort<_Tp> __select_radix_impl() noexcept
|
||||
{
|
||||
if constexpr (__is_one_of_v<remove_cvref_t<_BinaryPredicate>, less<>, less<_Tp>>)
|
||||
{
|
||||
return CUB_NS_QUALIFIER::DeviceRadixSort::SortKeys;
|
||||
}
|
||||
else
|
||||
{
|
||||
return CUB_NS_QUALIFIER::DeviceRadixSort::SortKeysDescending;
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Policy, class _Tp, class _BinaryPredicate>
|
||||
_CCCL_HOST_API static void __radix_sort_impl(const _Policy& __policy, _Tp* __first, _Tp* __last, _BinaryPredicate)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
const auto __count = static_cast<size_t>(::cuda::std::distance(__first, __last));
|
||||
|
||||
CUB_NS_QUALIFIER::DoubleBuffer<_Tp> __buffer{__first, nullptr};
|
||||
|
||||
constexpr _DeviceRadixSort<_Tp> __device_radix_sort = __select_radix_impl<_Tp, _BinaryPredicate>();
|
||||
|
||||
// Determine temporary device storage requirements for device_sort
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
__device_radix_sort,
|
||||
"__pstl_cuda_sort: determination of device storage for cub::DeviceRadixSort::SortKeys failed",
|
||||
static_cast<void*>(nullptr),
|
||||
__num_bytes,
|
||||
__buffer,
|
||||
__count,
|
||||
0,
|
||||
static_cast<int>(sizeof(_Tp) * CHAR_BIT),
|
||||
__stream.get());
|
||||
|
||||
{
|
||||
__temporary_storage<_Tp> __storage{__policy, __num_bytes, ::cuda::round_up(__count, 128)};
|
||||
__buffer.d_buffers[1] = __storage.template __get_raw_ptr<0>();
|
||||
|
||||
// Run the kernel
|
||||
_CCCL_TRY_CUDA_API(
|
||||
__device_radix_sort,
|
||||
"__pstl_cuda_sort: kernel launch of cub::DeviceRadixSort::SortKeys failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
__buffer,
|
||||
__count,
|
||||
0,
|
||||
static_cast<int>(sizeof(_Tp) * CHAR_BIT),
|
||||
__stream.get());
|
||||
|
||||
// Need to copy the memory back
|
||||
if (__buffer.selector != 0)
|
||||
{
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceTransform::TransformIf,
|
||||
"__pstl_cuda_sort: kernel launch of cub::DeviceTransform::TransformIf failed",
|
||||
tuple{__storage.template __get_raw_ptr<0>()},
|
||||
__first,
|
||||
__count,
|
||||
::cuda::always_true{},
|
||||
identity{},
|
||||
__stream.get());
|
||||
}
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
}
|
||||
|
||||
template <class _Policy, class _InputIterator, class _BinaryPredicate>
|
||||
_CCCL_HOST_API static void
|
||||
__merge_sort_impl(const _Policy& __policy, _InputIterator __first, _InputIterator __last, _BinaryPredicate __pred)
|
||||
{
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{::cudaStream_t{}}, __policy);
|
||||
|
||||
// Run the kernel
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceMergeSort::SortKeys,
|
||||
"__pstl_cuda_sort: kernel launch of cub::DeviceMergeSort::SortKeys failed",
|
||||
::cuda::std::move(__first),
|
||||
__count,
|
||||
::cuda::std::move(__pred),
|
||||
__policy);
|
||||
|
||||
__stream.sync();
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _BinaryPredicate)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator>)
|
||||
_CCCL_HOST_API void operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_BinaryPredicate __pred) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
if constexpr (CUB_NS_QUALIFIER::__can_use_radix_sort<_InputIterator, _BinaryPredicate> //
|
||||
&& __can_to_address<_InputIterator>)
|
||||
{
|
||||
__radix_sort_impl(
|
||||
__policy, ::cuda::std::to_address(__first), ::cuda::std::to_address(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
else
|
||||
{
|
||||
__merge_sort_impl(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "CUDA backend of cuda::std::sort requires random access iterators");
|
||||
// TODO(miscco) Implement a GPU friendly serial sort
|
||||
// ::cuda::std::sort(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_SORT_H
|
||||
@@ -0,0 +1,181 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_STABLE_PARTITION_H
|
||||
#define _CUDA_STD___PSTL_CUDA_STABLE_PARTITION_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_partition.cuh>
|
||||
# include <cub/device/device_transform.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/always_true_false.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/stable_partition.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/identity.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__pstl/reverse.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__stable_partition, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _UnaryPred>
|
||||
[[nodiscard]] _CCCL_HOST_API static _InputIterator
|
||||
__par_impl(const _Policy& __policy, _InputIterator __first, _InputIterator __last, _UnaryPred __pred)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
using _OffsetType = size_t;
|
||||
using value_type = iter_value_t<_InputIterator>;
|
||||
|
||||
_OffsetType __num_selected;
|
||||
const auto __count = static_cast<_OffsetType>(::cuda::std::distance(__first, __last));
|
||||
|
||||
// Determine temporary device storage requirements for device_stable_partition
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DevicePartition::If,
|
||||
"__pstl_cuda_stable_partition: determination of device storage for cub::DevicePartition::If failed",
|
||||
static_cast<void*>(nullptr),
|
||||
__num_bytes,
|
||||
static_cast<value_type*>(nullptr),
|
||||
__first,
|
||||
static_cast<_OffsetType*>(nullptr),
|
||||
__count,
|
||||
__pred,
|
||||
__policy);
|
||||
|
||||
{
|
||||
__temporary_storage<_OffsetType, value_type> __storage{__policy, __num_bytes, 1, __count};
|
||||
|
||||
// Partition cannot run inplace, so we need to first copy the input into temporary storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceTransform::TransformIf,
|
||||
"__pstl_cuda_stable_partition: kernel launch of cub::DeviceTransform::TransformIf failed",
|
||||
tuple<_InputIterator>{__first},
|
||||
__storage.template __get_ptr<1>(),
|
||||
__count,
|
||||
::cuda::always_true{},
|
||||
identity{},
|
||||
__policy);
|
||||
|
||||
// Run the kernel, the standard requires that the input and output range do not overlap
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DevicePartition::If,
|
||||
"__pstl_cuda_stable_partition: kernel launch of cub::DevicePartition::If failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
__storage.template __get_raw_ptr<1>(),
|
||||
__first,
|
||||
__storage.template __get_ptr<0>(),
|
||||
__count,
|
||||
::cuda::std::move(__pred),
|
||||
__policy);
|
||||
|
||||
// Copy the result back from storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
::cudaMemcpyAsync,
|
||||
"__pstl_cuda_stable_partition: copy of result from device to host failed",
|
||||
::cuda::std::addressof(__num_selected),
|
||||
__storage.template __get_ptr<0>(),
|
||||
sizeof(_OffsetType),
|
||||
::cudaMemcpyDefault,
|
||||
__stream.get());
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
|
||||
// Need to reverse the elements in the second partition
|
||||
const auto __mid = __first + static_cast<iter_difference_t<_InputIterator>>(__num_selected);
|
||||
::cuda::std::reverse(__policy, __mid, __last);
|
||||
return __mid;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _UnaryPred)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _InputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _UnaryPred __pred) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"CUDA backend of cuda::std::stable_partition requires random access iterators");
|
||||
return ::cuda::std::stable_partition(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_STABLE_PARTITION_H
|
||||
@@ -0,0 +1,209 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_TEMPORARY_STORAGE_H
|
||||
#define _CUDA_STD___PSTL_CUDA_TEMPORARY_STORAGE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/__cmath/round_up.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__iterator/tabulate_output_iterator.h>
|
||||
# include <cuda/__memory/align_up.h>
|
||||
# include <cuda/__memory_pool/device_memory_pool.h>
|
||||
# include <cuda/__memory_resource/any_resource.h>
|
||||
# include <cuda/__memory_resource/get_memory_resource.h>
|
||||
# include <cuda/__memory_resource/get_property.h>
|
||||
# include <cuda/__memory_resource/properties.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__memory/construct_at.h>
|
||||
# include <cuda/std/__type_traits/is_callable.h>
|
||||
# include <cuda/std/__type_traits/remove_cvref.h>
|
||||
# include <cuda/std/__type_traits/type_list.h>
|
||||
# include <cuda/std/__utility/forward.h>
|
||||
# include <cuda/std/__utility/integer_sequence.h>
|
||||
# include <cuda/std/cstdint>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
template <class _ResultType>
|
||||
struct __temporary_storage_construct_result
|
||||
{
|
||||
_ResultType* __result_;
|
||||
|
||||
_CCCL_HOST_API __temporary_storage_construct_result(_ResultType* __result = nullptr) noexcept
|
||||
: __result_(__result)
|
||||
{}
|
||||
|
||||
template <class _Index, class _Up>
|
||||
_CCCL_DEVICE_API _CCCL_FORCEINLINE void
|
||||
operator()(_Index, _Up&& __value) noexcept(is_nothrow_constructible_v<_ResultType, _Up>)
|
||||
{
|
||||
::cuda::std::__construct_at(__result_, ::cuda::std::forward<_Up>(__value));
|
||||
}
|
||||
};
|
||||
|
||||
//! @brief Provides device accessible storage for a number of typed sequences and temporary storage the algorithm might
|
||||
//! need.
|
||||
template <class... _StoredTypes>
|
||||
class __temporary_storage
|
||||
{
|
||||
::cuda::stream_ref __stream_;
|
||||
::cuda::mr::resource_ref<> __resource_;
|
||||
size_t __total_bytes_allocated_;
|
||||
array<void*, 1 + sizeof...(_StoredTypes)> __storage_;
|
||||
|
||||
_CCCL_TEMPLATE(class... _Sizes)
|
||||
_CCCL_REQUIRES((sizeof...(_Sizes) == sizeof...(_StoredTypes)))
|
||||
[[nodiscard]] _CCCL_HOST_API static constexpr size_t
|
||||
__get_total_bytes_allocated(const size_t __num_bytes_storage, const _Sizes... __elements_stored) noexcept
|
||||
{
|
||||
return (::cuda::round_up(static_cast<size_t>(__elements_stored) * sizeof(_StoredTypes),
|
||||
::cuda::mr::default_cuda_malloc_alignment)
|
||||
+ ... + ::cuda::round_up(__num_bytes_storage, ::cuda::mr::default_cuda_malloc_alignment));
|
||||
}
|
||||
|
||||
template <size_t _Index>
|
||||
[[nodiscard]] _CCCL_HOST_API static constexpr array<void*, 1 + sizeof...(_StoredTypes)>
|
||||
__get_storage(array<void*, 1 + sizeof...(_StoredTypes)>& __storage,
|
||||
const array<size_t, sizeof...(_StoredTypes)>& __num_elements) noexcept
|
||||
{
|
||||
if constexpr (_Index == sizeof...(_StoredTypes))
|
||||
{
|
||||
return __storage;
|
||||
}
|
||||
else
|
||||
{
|
||||
using _StoredType = __type_at_c<_Index, __type_list<_StoredTypes...>>;
|
||||
__storage[_Index + 1] = static_cast<void*>(
|
||||
::cuda::align_up(static_cast<_StoredType*>(__storage[_Index]) + __num_elements[_Index],
|
||||
::cuda::mr::default_cuda_malloc_alignment));
|
||||
return __get_storage<_Index + 1>(__storage, __num_elements);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class... _Sizes)
|
||||
_CCCL_REQUIRES((sizeof...(_Sizes) == sizeof...(_StoredTypes)))
|
||||
[[nodiscard]] _CCCL_HOST_API static constexpr array<void*, 1 + sizeof...(_StoredTypes)>
|
||||
__get_storage(void* __ptr, const _Sizes... __elements_stored) noexcept
|
||||
{
|
||||
array<void*, 1 + sizeof...(_StoredTypes)> __storage{__ptr};
|
||||
array<size_t, sizeof...(_StoredTypes)> __num_elements{static_cast<size_t>(__elements_stored)...};
|
||||
return __get_storage<0>(__storage, __num_elements);
|
||||
}
|
||||
|
||||
//! @brief Helper function to retrieve a memory resource from a policy
|
||||
//! In contrast to `__call_or` it does not require us to always call .device() on the stream
|
||||
template <class _Policy>
|
||||
[[nodiscard]] _CCCL_HOST_API static ::cuda::mr::resource_ref<> __get_memory_resource_or(const _Policy& __policy)
|
||||
{
|
||||
if constexpr (__is_callable_v<::cuda::mr::get_memory_resource_t, const _Policy&>)
|
||||
{
|
||||
const auto& __resource = ::cuda::mr::get_memory_resource(__policy);
|
||||
using __resource_t = remove_cvref_t<decltype(__resource)>;
|
||||
|
||||
if constexpr (!::cuda::mr::resource_with<::cuda::mr::device_accessible>
|
||||
&& ::cuda::has_property<decltype(__resource), ::cuda::mr::dynamic_accessibility_property>)
|
||||
{ //
|
||||
[[maybe_unused]] const ::cuda::mr::__memory_accessibility __dynamic_accessibility =
|
||||
get_property(__resource, ::cuda::mr::dynamic_accessibility_property{});
|
||||
_CCCL_ASSERT(__dynamic_accessibility == ::cuda::mr::__memory_accessibility::__device
|
||||
|| __dynamic_accessibility == ::cuda::mr::__memory_accessibility::__host_device,
|
||||
"Memory resources need to provide device accessible memory");
|
||||
}
|
||||
return ::cuda::mr::resource_ref<>{const_cast<__resource_t&>(__resource)};
|
||||
}
|
||||
else if constexpr (__is_callable_v<::cuda::get_stream_t, const _Policy&>)
|
||||
{
|
||||
return ::cuda::device_default_memory_pool(::cuda::get_stream(__policy).device());
|
||||
}
|
||||
else
|
||||
{
|
||||
// If no stream was specified, use the current device.
|
||||
int __curr_device{};
|
||||
_CCCL_TRY_CUDA_API(::cudaGetDevice, "Failed to get current device", &__curr_device);
|
||||
return ::cuda::device_default_memory_pool(__curr_device);
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
_CCCL_TEMPLATE(class _Policy, class... _Sizes)
|
||||
_CCCL_REQUIRES((sizeof...(_Sizes) == sizeof...(_StoredTypes)))
|
||||
_CCCL_HOST_API
|
||||
__temporary_storage(const _Policy& __policy, const size_t __num_bytes_storage, const _Sizes... __elements_stored)
|
||||
: __stream_(::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{::cudaStream_t{}}, __policy))
|
||||
, __resource_(__get_memory_resource_or(__policy))
|
||||
, __total_bytes_allocated_(__get_total_bytes_allocated(__num_bytes_storage, __elements_stored...))
|
||||
, __storage_(__get_storage(
|
||||
__resource_.allocate(__stream_, __total_bytes_allocated_, ::cuda::mr::default_cuda_malloc_alignment),
|
||||
__elements_stored...))
|
||||
{}
|
||||
|
||||
_CCCL_HOST_API ~__temporary_storage()
|
||||
{
|
||||
__resource_.deallocate(
|
||||
__stream_, __storage_[0], __total_bytes_allocated_, ::cuda::mr::default_cuda_malloc_alignment);
|
||||
}
|
||||
|
||||
//! We are dealing with uninitialized storage, so we might need to go through construct_at
|
||||
template <size_t _Index, class _OtherType = __type_at_c<_Index, __type_list<_StoredTypes...>>>
|
||||
[[nodiscard]] _CCCL_HOST_API auto __get_ptr() noexcept
|
||||
{
|
||||
static_assert(_Index < sizeof...(_StoredTypes), "__temporary_storage::__get_ptr: Invalid index");
|
||||
using _StoredType = __type_at_c<_Index, __type_list<_StoredTypes...>>;
|
||||
if constexpr (::cuda::std::__detail::__can_optimize_construct_at<_StoredType, _OtherType>)
|
||||
{
|
||||
return static_cast<_StoredType*>(__storage_[_Index]);
|
||||
}
|
||||
else
|
||||
{
|
||||
return ::cuda::tabulate_output_iterator{
|
||||
__temporary_storage_construct_result<_StoredType>{static_cast<_StoredType*>(__storage_[_Index])}};
|
||||
}
|
||||
}
|
||||
|
||||
//! When we know we can just return a plain pointer
|
||||
template <size_t _Index>
|
||||
[[nodiscard]] _CCCL_HOST_API auto* __get_raw_ptr() noexcept
|
||||
{
|
||||
static_assert(_Index < sizeof...(_StoredTypes), "__temporary_storage::__get_ptr: Invalid index");
|
||||
using _StoredType = __type_at_c<_Index, __type_list<_StoredTypes...>>;
|
||||
return static_cast<_StoredType*>(__storage_[_Index]);
|
||||
}
|
||||
|
||||
// The final pointer is always the temporary storage for the algorithm
|
||||
[[nodiscard]] _CCCL_HOST_API void* __get_temp_storage() noexcept
|
||||
{
|
||||
return __storage_[sizeof...(_StoredTypes)];
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_TEMPORARY_STORAGE_H
|
||||
@@ -0,0 +1,211 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_TRANSFORM_H
|
||||
#define _CUDA_STD___PSTL_CUDA_TRANSFORM_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_transform.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/always_true_false.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/transform.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__host_stdlib/new>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/tuple>
|
||||
|
||||
# include <cuda_runtime.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__transform, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _OutputIterator, class _UnaryOp, class... _InputIterators, class _Predicate>
|
||||
[[nodiscard]] _CCCL_HOST_API static _OutputIterator __par_impl(
|
||||
const _Policy& __policy,
|
||||
tuple<_InputIterators...> __first,
|
||||
_OutputIterator __result,
|
||||
const int64_t __count,
|
||||
_UnaryOp __func,
|
||||
_Predicate __pred)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
// We pass the policy as an environment to DeviceTransform
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceTransform::TransformIf,
|
||||
"cuda::std::transform: failed inside CUDA backend",
|
||||
::cuda::std::move(__first),
|
||||
__result,
|
||||
__count,
|
||||
::cuda::std::move(__pred),
|
||||
::cuda::std::move(__func),
|
||||
__policy);
|
||||
|
||||
__stream.sync();
|
||||
|
||||
return __result + __count;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(
|
||||
class _Policy, class _InputIterator, class _OutputIterator, class _UnaryOp, class _Predicate = ::cuda::always_true)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator> _CCCL_AND
|
||||
is_invocable_v<_UnaryOp, iter_reference_t<_InputIterator>>)
|
||||
[[nodiscard]] _CCCL_HOST_API _OutputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_UnaryOp __func,
|
||||
_Predicate __pred = {}) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>
|
||||
&& ::cuda::std::__has_random_access_traversal<_OutputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
return __par_impl(
|
||||
__policy,
|
||||
::cuda::std::make_tuple(::cuda::std::move(__first)),
|
||||
::cuda::std::move(__result),
|
||||
__count,
|
||||
::cuda::std::move(__func),
|
||||
::cuda::std::move(__pred));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::transform requires at least random access iterators");
|
||||
return ::cuda::std::transform(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__result), ::cuda::std::move(__func));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy,
|
||||
class _InputIterator1,
|
||||
class _InputIterator2,
|
||||
class _OutputIterator,
|
||||
class _BinaryOp,
|
||||
class _Predicate = ::cuda::always_true)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator1> _CCCL_AND __has_forward_traversal<_InputIterator2> _CCCL_AND
|
||||
__has_forward_traversal<_OutputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _OutputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator1 __first1,
|
||||
_InputIterator1 __last1,
|
||||
_InputIterator2 __first2,
|
||||
_OutputIterator __result,
|
||||
_BinaryOp __func,
|
||||
_Predicate __pred = {}) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator1>
|
||||
&& ::cuda::std::__has_random_access_traversal<_InputIterator2>
|
||||
&& ::cuda::std::__has_random_access_traversal<_OutputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
const auto __count = ::cuda::std::distance(__first1, __last1);
|
||||
return __par_impl(
|
||||
__policy,
|
||||
::cuda::std::make_tuple(::cuda::std::move(__first1), ::cuda::std::move(__first2)),
|
||||
::cuda::std::move(__result),
|
||||
__count,
|
||||
::cuda::std::move(__func),
|
||||
::cuda::std::move(__pred));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::transform requires at least random access iterators");
|
||||
return ::cuda::std::transform(
|
||||
::cuda::std::move(__first1),
|
||||
::cuda::std::move(__last1),
|
||||
::cuda::std::move(__first2),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__func));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_TRANSFORM_H
|
||||
@@ -0,0 +1,190 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_TRANSFORM_REDUCE_H
|
||||
#define _CUDA_STD___PSTL_CUDA_TRANSFORM_REDUCE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wignored-attributes")
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_reduce.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__functional/call_or.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__iterator/next.h>
|
||||
# include <cuda/std/__memory/addressof.h>
|
||||
# include <cuda/std/__memory/construct_at.h>
|
||||
# include <cuda/std/__numeric/transform_reduce.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/tuple>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__transform_reduce, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _Size, class _Tp, class _ReductionOp, class _TransformOp>
|
||||
[[nodiscard]] _CCCL_HOST_API static _Tp __par_impl(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_Size __count,
|
||||
_Tp __init,
|
||||
_ReductionOp __reduction_op,
|
||||
_TransformOp __transform_op)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
|
||||
_Tp __ret;
|
||||
|
||||
// We need to know the accumulator type to determine whether we need construct_at for the return value
|
||||
using _AccumT = __accumulator_t<_ReductionOp, invoke_result_t<_TransformOp, iter_reference_t<_InputIterator>>, _Tp>;
|
||||
|
||||
// Determine temporary device storage requirements for reduce
|
||||
void* __temp_storage = nullptr;
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceReduce::TransformReduce,
|
||||
"__pstl_cuda_transform_reduce: determination of device storage for cub::DeviceReduce::TransformReduce failed",
|
||||
__temp_storage,
|
||||
__num_bytes,
|
||||
__first,
|
||||
static_cast<_Tp*>(nullptr),
|
||||
__count,
|
||||
__reduction_op,
|
||||
__transform_op,
|
||||
__init);
|
||||
|
||||
{
|
||||
__temporary_storage<_Tp> __storage{__policy, __num_bytes, 1};
|
||||
|
||||
// Run the reduction
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceReduce::TransformReduce,
|
||||
"__pstl_cuda_transform_reduce: kernel launch of cub::DeviceReduce::TransformReduce failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
::cuda::std::move(__first),
|
||||
__storage.template __get_ptr<0, _AccumT>(),
|
||||
__count,
|
||||
::cuda::std::move(__reduction_op),
|
||||
::cuda::std::move(__transform_op),
|
||||
::cuda::std::move(__init),
|
||||
__stream.get());
|
||||
|
||||
// Copy the result back from storage
|
||||
_CCCL_TRY_CUDA_API(
|
||||
::cudaMemcpyAsync,
|
||||
"__pstl_cuda_transformm_reduce: copy of result from device to host failed",
|
||||
::cuda::std::addressof(__ret),
|
||||
__storage.template __get_ptr<0>(),
|
||||
sizeof(_Tp),
|
||||
::cudaMemcpyDefault,
|
||||
__stream.get());
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
return __ret;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _Size, class _Tp, class _ReductionOp, class _TransformOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _Tp operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_Size __count,
|
||||
_Tp __init,
|
||||
_ReductionOp __reduction_op,
|
||||
_TransformOp __transform_op) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
__count,
|
||||
::cuda::std::move(__init),
|
||||
::cuda::std::move(__reduction_op),
|
||||
::cuda::std::move(__transform_op));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == ::cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::transform_reduce requires at least random access "
|
||||
"iterators");
|
||||
auto __last = ::cuda::std::next(__first, __count);
|
||||
return ::cuda::std::transform_reduce(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__init),
|
||||
::cuda::std::move(__reduction_op),
|
||||
::cuda::std::move(__transform_op));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif /// _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_TRANSFORM_REDUCE_H
|
||||
162
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/unique.h
Normal file
162
cccl_upstream/libcudacxx/include/cuda/std/__pstl/cuda/unique.h
Normal file
@@ -0,0 +1,162 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_UNIQUE_H
|
||||
#define _CUDA_STD___PSTL_CUDA_UNIQUE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_select.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/unique.h>
|
||||
# include <cuda/std/__algorithm/unique_copy.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__iterator/next.h>
|
||||
# include <cuda/std/__memory/pointer_traits.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/cstdint>
|
||||
# include <cuda/std/tuple>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__unique, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _BinaryPredicate>
|
||||
[[nodiscard]] _CCCL_HOST_API static _InputIterator
|
||||
__par_impl(const _Policy& __policy, _InputIterator __first, _InputIterator __last, _BinaryPredicate __pred)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
|
||||
using _OffsetType = iter_difference_t<_InputIterator>;
|
||||
_OffsetType __num_selected = 0;
|
||||
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceSelect::Unique,
|
||||
"__pstl_cuda_unique: determination of device storage for cub::DeviceSelect::Unique failed",
|
||||
static_cast<void*>(nullptr),
|
||||
__num_bytes,
|
||||
__first,
|
||||
static_cast<_OffsetType*>(nullptr),
|
||||
__count,
|
||||
__pred,
|
||||
__policy);
|
||||
|
||||
{ // Create temporary storage for the return value (num_selected) and CUB internal scratch space
|
||||
__temporary_storage<_OffsetType> __storage{__policy, __num_bytes, 1};
|
||||
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceSelect::Unique,
|
||||
"__pstl_cuda_unique: kernel launch of cub::DeviceSelect::Unique failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
__first,
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
__count,
|
||||
::cuda::std::move(__pred),
|
||||
__policy);
|
||||
|
||||
_CCCL_TRY_CUDA_API(
|
||||
::cudaMemcpyAsync,
|
||||
"__pstl_cuda_unique: copy of num_selected from device to host failed",
|
||||
::cuda::std::addressof(__num_selected),
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
sizeof(_OffsetType),
|
||||
cudaMemcpyDefault,
|
||||
__stream.get());
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
return __first + __num_selected;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _BinaryPredicate)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _InputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_BinaryPredicate __pred) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(__policy, __first, ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == ::cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::unique requires at least random access iterators");
|
||||
return ::cuda::std::unique(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_UNIQUE_H
|
||||
@@ -0,0 +1,177 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_CUDA_UNIQUE_COPY_H
|
||||
#define _CUDA_STD___PSTL_CUDA_UNIQUE_COPY_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wshadow")
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
|
||||
_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
|
||||
|
||||
# include <cub/device/device_select.cuh>
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# include <cuda/__execution/policy.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__stream/get_stream.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__algorithm/unique.h>
|
||||
# include <cuda/std/__algorithm/unique_copy.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__execution/env.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__iterator/next.h>
|
||||
# include <cuda/std/__memory/pointer_traits.h>
|
||||
# include <cuda/std/__pstl/cuda/ensure_current_context.h>
|
||||
# include <cuda/std/__pstl/cuda/temporary_storage.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/cstdint>
|
||||
# include <cuda/std/tuple>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <>
|
||||
struct __pstl_dispatch<__pstl_algorithm::__unique_copy, __execution_backend::__cuda>
|
||||
{
|
||||
template <class _Policy, class _InputIterator, class _OutputIterator, class _BinaryPredicate>
|
||||
[[nodiscard]] _CCCL_HOST_API static _OutputIterator __par_impl(
|
||||
const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_BinaryPredicate __pred)
|
||||
{
|
||||
const auto __stream = ::cuda::__call_or(::cuda::get_stream, ::cuda::stream_ref{cudaStream_t{}}, __policy);
|
||||
const auto __ctx = ::cuda::std::execution::__pstl_ensure_current_ctx_for(__policy);
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
|
||||
using _OffsetType = iter_difference_t<_InputIterator>;
|
||||
_OffsetType __num_selected = 0;
|
||||
|
||||
size_t __num_bytes = 0;
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceSelect::Unique,
|
||||
"__pstl_cuda_unique: determination of device storage for cub::DeviceSelect::Unique failed",
|
||||
static_cast<void*>(nullptr),
|
||||
__num_bytes,
|
||||
__first,
|
||||
__result,
|
||||
static_cast<_OffsetType*>(nullptr),
|
||||
__count,
|
||||
__pred,
|
||||
__policy);
|
||||
|
||||
{ // Create temporary storage for the return value (num_selected) and CUB internal scratch space
|
||||
__temporary_storage<_OffsetType> __storage{__policy, __num_bytes, 1};
|
||||
|
||||
_CCCL_TRY_CUDA_API(
|
||||
CUB_NS_QUALIFIER::DeviceSelect::Unique,
|
||||
"__pstl_cuda_unique: kernel launch of cub::DeviceSelect::Unique failed",
|
||||
__storage.__get_temp_storage(),
|
||||
__num_bytes,
|
||||
::cuda::std::move(__first),
|
||||
__result,
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
__count,
|
||||
::cuda::std::move(__pred),
|
||||
__policy);
|
||||
|
||||
_CCCL_TRY_CUDA_API(
|
||||
::cudaMemcpyAsync,
|
||||
"__pstl_cuda_unique: copy of num_selected from device to host failed",
|
||||
::cuda::std::addressof(__num_selected),
|
||||
__storage.template __get_raw_ptr<0>(),
|
||||
sizeof(_OffsetType),
|
||||
cudaMemcpyDefault,
|
||||
__stream.get());
|
||||
}
|
||||
|
||||
__stream.sync();
|
||||
return __result + static_cast<iter_difference_t<_OutputIterator>>(__num_selected);
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _BinaryPredicate)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_OutputIterator>)
|
||||
[[nodiscard]] _CCCL_HOST_API _OutputIterator operator()(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_BinaryPredicate __pred) const
|
||||
{
|
||||
if constexpr (::cuda::std::__has_random_access_traversal<_InputIterator>
|
||||
&& ::cuda::std::__has_random_access_traversal<_OutputIterator>)
|
||||
{
|
||||
_CCCL_TRY
|
||||
{
|
||||
return __par_impl(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__pred));
|
||||
}
|
||||
_CCCL_CATCH (const ::cuda::cuda_error& __err)
|
||||
{
|
||||
if (__err.status() == ::cudaErrorMemoryAllocation)
|
||||
{
|
||||
_CCCL_THROW(::std::bad_alloc);
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_RETHROW;
|
||||
}
|
||||
}
|
||||
_CCCL_CATCH_FALLTHROUGH
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"__pstl_dispatch: CUDA backend of cuda::std::unique_copy requires at least random access "
|
||||
"iterators");
|
||||
return ::cuda::std::unique_copy(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__result), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_CUDA_UNIQUE_COPY_H
|
||||
112
cccl_upstream/libcudacxx/include/cuda/std/__pstl/dispatch.h
Normal file
112
cccl_upstream/libcudacxx/include/cuda/std/__pstl/dispatch.h
Normal file
@@ -0,0 +1,112 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_DISPATCH_H
|
||||
#define _CUDA_STD___PSTL_DISPATCH_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__execution/policy.h>
|
||||
#include <cuda/std/__type_traits/always_false.h>
|
||||
#include <cuda/std/__type_traits/is_base_of.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
enum class __pstl_algorithm
|
||||
{
|
||||
__adjacent_difference,
|
||||
__copy_if,
|
||||
__copy_n,
|
||||
__exclusive_scan,
|
||||
__find_if,
|
||||
__for_each_n,
|
||||
__generate_n,
|
||||
__inclusive_scan,
|
||||
__max_element,
|
||||
__merge,
|
||||
__min_element,
|
||||
__partition,
|
||||
__partition_copy,
|
||||
__reduce,
|
||||
__remove_if,
|
||||
__rotate,
|
||||
__rotate_copy,
|
||||
__shift_left,
|
||||
__shift_right,
|
||||
__stable_partition,
|
||||
__sort,
|
||||
__transform,
|
||||
__transform_reduce,
|
||||
__unique,
|
||||
__unique_copy,
|
||||
};
|
||||
|
||||
//! @brief tag type to indicate that we cannot dispatch to a parallel algorithm and should run the algorithm serially
|
||||
struct __pstl_no_dispatch
|
||||
{};
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
//! @brief Dispatcher for a given @tparam _Algorith and @tparam _Policy
|
||||
//! If @class __pstl_dispatch is not specialized by the chosen backend we will fall back to serial execution
|
||||
template <__pstl_algorithm _Algorithm, __execution_backend _Backend>
|
||||
struct __pstl_dispatch : public __pstl_no_dispatch
|
||||
{};
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
//! @brief Helper variable that detects whether @class __pstl_dispatch has been specialized so that we can
|
||||
//! dispatch
|
||||
template <class>
|
||||
inline constexpr bool __pstl_can_dispatch = false;
|
||||
|
||||
template <__pstl_algorithm _Algorithm, __execution_backend _Backend>
|
||||
inline constexpr bool __pstl_can_dispatch<__pstl_dispatch<_Algorithm, _Backend>> =
|
||||
!::cuda::std::is_base_of_v<__pstl_no_dispatch, __pstl_dispatch<_Algorithm, _Backend>>;
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
//! @brief Top layer dispatcher that returns a concrete dispatch if possible
|
||||
template <__pstl_algorithm _Algorithm, class _Policy>
|
||||
[[nodiscard]] _CCCL_HOST_API _CCCL_CONSTEVAL auto __pstl_select_dispatch() noexcept
|
||||
{
|
||||
// First extract the desired backend from the policy
|
||||
constexpr __execution_backend __backend = _Policy::__get_backend();
|
||||
|
||||
// If the user requests a unique backends, we must take that
|
||||
if constexpr (::cuda::std::execution::__has_unique_backend(__backend))
|
||||
{
|
||||
return __pstl_dispatch<_Algorithm, __backend>{};
|
||||
}
|
||||
else
|
||||
{
|
||||
// No dispatch found, return invalid to signal serial execution
|
||||
return __pstl_dispatch<_Algorithm, __execution_backend::__none>{};
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_STD___PSTL_DISPATCH_H
|
||||
156
cccl_upstream/libcudacxx/include/cuda/std/__pstl/equal.h
Normal file
156
cccl_upstream/libcudacxx/include/cuda/std/__pstl/equal.h
Normal file
@@ -0,0 +1,156 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_EQUAL_H
|
||||
#define _CUDA_STD___PSTL_EQUAL_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/zip_function.h>
|
||||
# include <cuda/__iterator/zip_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/equal.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/not_fn.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_comparable.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/find_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIter1, class _InputIter2, class _BinaryPred = ::cuda::std::equal_to<>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIter1> _CCCL_AND __has_forward_traversal<_InputIter2> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API bool
|
||||
equal([[maybe_unused]] const _Policy& __policy,
|
||||
_InputIter1 __first1,
|
||||
_InputIter1 __last1,
|
||||
_InputIter2 __first2,
|
||||
_BinaryPred __pred = {})
|
||||
{
|
||||
static_assert(indirect_binary_predicate<_BinaryPred, _InputIter1, _InputIter2>,
|
||||
"cuda::std::equal: BinaryPred must satisfy "
|
||||
"indirect_binary_predicate<BinaryPred, InputIter1, InputIter2>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::equal");
|
||||
|
||||
if (__first1 == __last1)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first1, __last1);
|
||||
auto __zip_first = ::cuda::zip_iterator{::cuda::std::move(__first1), ::cuda::std::move(__first2)};
|
||||
const auto __zip_last = __zip_first + __count;
|
||||
const auto __result = __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__zip_first),
|
||||
__zip_last,
|
||||
::cuda::zip_function{::cuda::std::not_fn(::cuda::std::move(__pred))});
|
||||
return __result == __zip_last;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::equal requires at least one selected backend");
|
||||
return ::cuda::std::equal(
|
||||
::cuda::std::move(__first1), ::cuda::std::move(__last1), ::cuda::std::move(__first2), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIter1, class _InputIter2, class _BinaryPred = ::cuda::std::equal_to<>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIter1> _CCCL_AND __has_forward_traversal<_InputIter2> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API bool equal(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIter1 __first1,
|
||||
_InputIter1 __last1,
|
||||
_InputIter2 __first2,
|
||||
_InputIter2 __last2,
|
||||
_BinaryPred __pred = {})
|
||||
{
|
||||
static_assert(indirect_binary_predicate<_BinaryPred, _InputIter1, _InputIter2>,
|
||||
"cuda::std::equal: BinaryPred must satisfy "
|
||||
"indirect_binary_predicate<BinaryPred, InputIter1, InputIter2>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::equal");
|
||||
|
||||
if (__first1 == __last1 && __first2 == __last2)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
const auto __count1 = ::cuda::std::distance(__first1, __last1);
|
||||
const auto __count2 = ::cuda::std::distance(__first2, __last2);
|
||||
if (__count1 != __count2)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
auto __zip_first = ::cuda::zip_iterator{::cuda::std::move(__first1), ::cuda::std::move(__first2)};
|
||||
const auto __zip_last = ::cuda::zip_iterator{::cuda::std::move(__last1), ::cuda::std::move(__last2)};
|
||||
const auto __result = __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__zip_first),
|
||||
__zip_last,
|
||||
::cuda::zip_function{::cuda::std::not_fn(::cuda::std::move(__pred))});
|
||||
return __result == __zip_last;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::equal requires at least one selected backend");
|
||||
return ::cuda::std::equal(
|
||||
::cuda::std::move(__first1),
|
||||
::cuda::std::move(__last1),
|
||||
::cuda::std::move(__first2),
|
||||
::cuda::std::move(__last2),
|
||||
::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_EQUAL_H
|
||||
@@ -0,0 +1,126 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_EXCLUSIVE_SCAN_H
|
||||
#define _CUDA_STD___PSTL_EXCLUSIVE_SCAN_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/invoke.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__numeric/exclusive_scan.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/exclusive_scan.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator exclusive_scan(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_Tp __init,
|
||||
_BinaryOp __binary_op)
|
||||
{
|
||||
static_assert(is_invocable_v<_BinaryOp, iter_reference_t<_InputIterator>, iter_reference_t<_InputIterator>>,
|
||||
"cuda::std::exclusive_scan requires UnaryOp to be invocable with "
|
||||
"iter_reference_t<InputIterator>, iter_reference_t<InputIterator>");
|
||||
|
||||
static_assert(
|
||||
indirectly_writable<_OutputIterator,
|
||||
invoke_result_t<_BinaryOp, iter_reference_t<_InputIterator>, iter_reference_t<_InputIterator>>>,
|
||||
"cuda::std::exclusive_scan requires OutputIterator to be indirectly writable with the return value of BinaryOp");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__exclusive_scan, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::exclusive_scan");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__init),
|
||||
::cuda::std::move(__binary_op));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::exclusive_scan requires at least one selected backend");
|
||||
::cuda::std::exclusive_scan(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__init),
|
||||
::cuda::std::move(__binary_op));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _Tp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator exclusive_scan(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_Tp __init)
|
||||
{
|
||||
return ::cuda::std::exclusive_scan(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__init),
|
||||
::cuda::std::plus<>{});
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_EXCLUSIVE_SCAN_H
|
||||
102
cccl_upstream/libcudacxx/include/cuda/std/__pstl/fill.h
Normal file
102
cccl_upstream/libcudacxx/include/cuda/std/__pstl/fill.h
Normal file
@@ -0,0 +1,102 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_FILL_H
|
||||
#define _CUDA_STD___PSTL_FILL_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/counting_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/fill.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/generate_n.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
template <class _Tp>
|
||||
struct __fill_constant_value
|
||||
{
|
||||
_Tp __val_;
|
||||
|
||||
_CCCL_API constexpr __fill_constant_value(const _Tp& __val) noexcept(is_nothrow_copy_constructible_v<_Tp>)
|
||||
: __val_(__val)
|
||||
{}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE constexpr const _Tp& operator()() const noexcept
|
||||
{
|
||||
return __val_;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _Tp = iter_value_t<_InputIterator>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API void
|
||||
fill([[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, const _Tp& __value)
|
||||
{
|
||||
static_assert(indirectly_writable<_InputIterator, const _Tp&>,
|
||||
"cuda::std::fill requires InputIterator to be indirectly writable with const T&");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__generate_n, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::fill");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// We do not want actually load anything, so pass a counting iterator instead
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
(void) __dispatch(__policy, ::cuda::std::move(__first), __count, __fill_constant_value{__value});
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::fill requires at least one selected backend");
|
||||
::cuda::std::fill(::cuda::std::move(__first), ::cuda::std::move(__last), __value);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_FILL_H
|
||||
85
cccl_upstream/libcudacxx/include/cuda/std/__pstl/fill_n.h
Normal file
85
cccl_upstream/libcudacxx/include/cuda/std/__pstl/fill_n.h
Normal file
@@ -0,0 +1,85 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_FILL_N_H
|
||||
#define _CUDA_STD___PSTL_FILL_N_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/counting_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/fill_n.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__pstl/fill.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/generate_n.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _Size, class _Tp = iter_value_t<_InputIterator>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _InputIterator
|
||||
fill_n([[maybe_unused]] const _Policy& __policy, _InputIterator __first, _Size __count, const _Tp& __value)
|
||||
{
|
||||
static_assert(indirectly_writable<_InputIterator, const _Tp&>,
|
||||
"cuda::std::fill_n requires InputIterator to be indirectly writable with const T&");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__generate_n, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::fill_n");
|
||||
|
||||
if (__count == 0)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
return __dispatch(__policy, ::cuda::std::move(__first), __count, __fill_constant_value{__value});
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::fill_n requires at least one selected backend");
|
||||
return ::cuda::std::fill_n(::cuda::std::move(__first), __count, __value);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_FILL_N_H
|
||||
87
cccl_upstream/libcudacxx/include/cuda/std/__pstl/find.h
Normal file
87
cccl_upstream/libcudacxx/include/cuda/std/__pstl/find.h
Normal file
@@ -0,0 +1,87 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_FIND_H
|
||||
#define _CUDA_STD___PSTL_FIND_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__functional/equal_to_value.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/find.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_comparable.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/find_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _Iter, class _Tp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_Iter> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API _Iter
|
||||
find([[maybe_unused]] const _Policy& __policy, _Iter __first, _Iter __last, const _Tp& __val)
|
||||
{
|
||||
static_assert(__is_cpp17_equality_comparable_v<_Tp, iter_value_t<_Iter>>,
|
||||
"Parallel cuda::std::find requires that T is equality comparable with iter_value_t<Iter>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::find");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::equal_to_value<_Tp>{__val});
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::find requires at least one selected backend");
|
||||
return ::cuda::std::find(::cuda::std::move(__first), ::cuda::std::move(__last), __val);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_FIND_H
|
||||
83
cccl_upstream/libcudacxx/include/cuda/std/__pstl/find_if.h
Normal file
83
cccl_upstream/libcudacxx/include/cuda/std/__pstl/find_if.h
Normal file
@@ -0,0 +1,83 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_FIND_IF_H
|
||||
#define _CUDA_STD___PSTL_FIND_IF_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/find_if.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/find_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _Iter, class _UnaryOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_Iter> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API _Iter
|
||||
find_if([[maybe_unused]] const _Policy& __policy, _Iter __first, _Iter __last, _UnaryOp __pred)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryOp, _Iter>,
|
||||
"cuda::std::find_if: UnaryOp must satisfy indirect_unary_predicate<Iter>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::find_if");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
return __dispatch(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::find_if requires at least one selected backend");
|
||||
return ::cuda::std::find_if(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_FIND_IF_H
|
||||
@@ -0,0 +1,85 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_FIND_IF_NOT_H
|
||||
#define _CUDA_STD___PSTL_FIND_IF_NOT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/find_if_not.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/not_fn.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/find_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _Iter, class _UnaryOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_Iter> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API _Iter
|
||||
find_if_not([[maybe_unused]] const _Policy& __policy, _Iter __first, _Iter __last, _UnaryOp __pred)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryOp, _Iter>,
|
||||
"cuda::std::find_if: UnaryOp must satisfy indirect_unary_predicate<Iter>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::find_if_not");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::not_fn(::cuda::std::move(__pred)));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::find_if_not requires at least one selected backend");
|
||||
return ::cuda::std::find_if_not(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_FIND_IF_NOT_H
|
||||
78
cccl_upstream/libcudacxx/include/cuda/std/__pstl/for_each.h
Normal file
78
cccl_upstream/libcudacxx/include/cuda/std/__pstl/for_each.h
Normal file
@@ -0,0 +1,78 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_FOR_EACH_H
|
||||
#define _CUDA_STD___PSTL_FOR_EACH_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/for_each_n.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/for_each_n.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _Iter, class _Fn)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_Iter> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API void for_each([[maybe_unused]] const _Policy& __policy, _Iter __first, _Iter __last, _Fn __func)
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__for_each_n, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::for_each");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
(void) __dispatch(__policy, ::cuda::std::move(__first), __count, ::cuda::std::move(__func));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::for_each requires at least one selected backend");
|
||||
::cuda::std::for_each(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__func));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_FOR_EACH_H
|
||||
@@ -0,0 +1,77 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_FOR_EACH_N_H
|
||||
#define _CUDA_STD___PSTL_FOR_EACH_N_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/for_each_n.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/for_each_n.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _Iter, class _Size, class _Fn)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_Iter> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _Iter for_each_n([[maybe_unused]] const _Policy& __policy, _Iter __first, _Size __orig_n, _Fn __func)
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__for_each_n, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::for_each_n");
|
||||
|
||||
if (__orig_n == 0)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
return __dispatch(__policy, ::cuda::std::move(__first), __orig_n, ::cuda::std::move(__func));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::for_each_n requires at least one selected backend");
|
||||
return ::cuda::std::for_each_n(::cuda::std::move(__first), __orig_n, ::cuda::std::move(__func));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_FOR_EACH_N_H
|
||||
88
cccl_upstream/libcudacxx/include/cuda/std/__pstl/generate.h
Normal file
88
cccl_upstream/libcudacxx/include/cuda/std/__pstl/generate.h
Normal file
@@ -0,0 +1,88 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_GENERATE_H
|
||||
#define _CUDA_STD___PSTL_GENERATE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/counting_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/generate.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_callable.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/generate_n.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _Generator)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API void
|
||||
generate([[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _Generator __gen)
|
||||
{
|
||||
static_assert(indirectly_writable<_InputIterator, invoke_result_t<_Generator>>,
|
||||
"cuda::std::generate requires InputIterator to be indirectly writable with the return value of "
|
||||
"Generator");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__generate_n, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::generate");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
(void) __dispatch(__policy, ::cuda::std::move(__first), __count, ::cuda::std::move(__gen));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::generate requires at least one selected backend");
|
||||
::cuda::std::generate(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__gen));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_GENERATE_H
|
||||
@@ -0,0 +1,87 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_GENERATE_N_H
|
||||
#define _CUDA_STD___PSTL_GENERATE_N_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/counting_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/generate_n.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__pstl/generate.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_callable.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/generate_n.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _Size, class _Generator)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _InputIterator
|
||||
generate_n([[maybe_unused]] const _Policy& __policy, _InputIterator __first, _Size __count, _Generator __gen)
|
||||
{
|
||||
static_assert(indirectly_writable<_InputIterator, invoke_result_t<_Generator>>,
|
||||
"cuda::std::generate_n requires InputIterator to be indirectly writable with the return value of "
|
||||
"Generator");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__generate_n, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::generate_n");
|
||||
|
||||
if (__count == 0)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
return __dispatch(__policy, ::cuda::std::move(__first), __count, ::cuda::std::move(__gen));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::generate_n requires at least one selected backend");
|
||||
return ::cuda::std::generate_n(::cuda::std::move(__first), __count, ::cuda::std::move(__gen));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_GENERATE_N_H
|
||||
@@ -0,0 +1,165 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_INCLUSIVE_SCAN_H
|
||||
#define _CUDA_STD___PSTL_INCLUSIVE_SCAN_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/invoke.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__numeric/inclusive_scan.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/inclusive_scan.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator inclusive_scan(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_BinaryOp __binary_op,
|
||||
_Tp __init)
|
||||
{
|
||||
static_assert(is_invocable_v<_BinaryOp, iter_reference_t<_InputIterator>, iter_reference_t<_InputIterator>>,
|
||||
"cuda::std::inclusive_scan requires UnaryOp to be invocable with "
|
||||
"iter_reference_t<InputIterator>, iter_reference_t<InputIterator>");
|
||||
|
||||
static_assert(
|
||||
indirectly_writable<_OutputIterator,
|
||||
invoke_result_t<_BinaryOp, iter_reference_t<_InputIterator>, iter_reference_t<_InputIterator>>>,
|
||||
"cuda::std::inclusive_scan requires OutputIterator to be indirectly writable with the return value of BinaryOp");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__inclusive_scan, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::inclusive_scan");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op),
|
||||
::cuda::std::move(__init));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::inclusive_scan requires at least one selected backend");
|
||||
return ::cuda::std::inclusive_scan(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op),
|
||||
::cuda::std::move(__init));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _BinaryOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator inclusive_scan(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_BinaryOp __binary_op)
|
||||
{
|
||||
static_assert(is_invocable_v<_BinaryOp, iter_reference_t<_InputIterator>, iter_reference_t<_InputIterator>>,
|
||||
"cuda::std::inclusive_scan requires UnaryOp to be invocable with "
|
||||
"iter_reference_t<InputIterator>, iter_reference_t<InputIterator>");
|
||||
|
||||
static_assert(
|
||||
indirectly_writable<_OutputIterator,
|
||||
invoke_result_t<_BinaryOp, iter_reference_t<_InputIterator>, iter_reference_t<_InputIterator>>>,
|
||||
"cuda::std::inclusive_scan requires OutputIterator to be indirectly writable with the return value of BinaryOp");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__inclusive_scan, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::inclusive_scan");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::inclusive_scan requires at least one selected backend");
|
||||
return ::cuda::std::inclusive_scan(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator inclusive_scan(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _OutputIterator __result)
|
||||
{
|
||||
return ::cuda::std::inclusive_scan(
|
||||
__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__result), ::cuda::std::plus<>{});
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_INCLUSIVE_SCAN_H
|
||||
104
cccl_upstream/libcudacxx/include/cuda/std/__pstl/is_heap.h
Normal file
104
cccl_upstream/libcudacxx/include/cuda/std/__pstl/is_heap.h
Normal file
@@ -0,0 +1,104 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_IS_HEAP_H
|
||||
#define _CUDA_STD___PSTL_IS_HEAP_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
# include <cuda/__iterator/counting_iterator.h>
|
||||
# include <cuda/__iterator/zip_function.h>
|
||||
# include <cuda/__iterator/zip_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/is_heap.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__pstl/is_heap_until.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/find_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _RandomAccessIterator, class _Compare = less<>)
|
||||
_CCCL_REQUIRES(__has_random_access_traversal<_RandomAccessIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API bool is_heap(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_RandomAccessIterator __first,
|
||||
_RandomAccessIterator __last,
|
||||
_Compare __comp = {})
|
||||
{
|
||||
static_assert(indirect_binary_predicate<_Compare, _RandomAccessIterator, _RandomAccessIterator>,
|
||||
"cuda::std::is_heap: Compare must satisfy "
|
||||
"indirect_binary_predicate<Compare, RandomAccessIterator, RandomAccessIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::is_heap");
|
||||
|
||||
using __diff_t = iter_difference_t<_RandomAccessIterator>;
|
||||
const auto __n = ::cuda::std::distance(__first, __last);
|
||||
if (__n < __diff_t(2))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// Find the first heap-property violation in the index range [1, n).
|
||||
// The result's dereferenced value is the violating child index, or n
|
||||
// if the range is a heap. `::cuda::std::get<1>(__result.__iterators())`
|
||||
// returns __last in the latter case.
|
||||
const auto __result = __dispatch(
|
||||
__policy,
|
||||
::cuda::zip_iterator{::cuda::counting_iterator{__diff_t(1)}, __first + 1},
|
||||
::cuda::zip_iterator{::cuda::counting_iterator{__n}, __last},
|
||||
::cuda::zip_function{__is_heap_until_fn<_RandomAccessIterator, _Compare>{__first, ::cuda::std::move(__comp)}});
|
||||
return ::cuda::std::get<1>(__result.__iterators()) == __last;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::is_heap requires at least one selected backend");
|
||||
return ::cuda::std::is_heap(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__comp));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
#endif // _CUDA_STD___PSTL_IS_HEAP_H
|
||||
124
cccl_upstream/libcudacxx/include/cuda/std/__pstl/is_heap_until.h
Normal file
124
cccl_upstream/libcudacxx/include/cuda/std/__pstl/is_heap_until.h
Normal file
@@ -0,0 +1,124 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_IS_HEAP_UNTIL_H
|
||||
#define _CUDA_STD___PSTL_IS_HEAP_UNTIL_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
# include <cuda/__iterator/counting_iterator.h>
|
||||
# include <cuda/__iterator/zip_function.h>
|
||||
# include <cuda/__iterator/zip_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/is_heap_until.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/find_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
// Returns true at the first child index `i` (1 <= i < n) where the parent-child
|
||||
// max-heap invariant is broken, i.e. comp(base[(i-1)/2], base[i]) holds.
|
||||
template <class _RandomAccessIterator, class _Compare>
|
||||
struct __is_heap_until_fn
|
||||
{
|
||||
_RandomAccessIterator __base_;
|
||||
_Compare __comp_;
|
||||
|
||||
_CCCL_API explicit constexpr __is_heap_until_fn(_RandomAccessIterator __base, _Compare __comp)
|
||||
: __base_(::cuda::std::move(__base))
|
||||
, __comp_(::cuda::std::move(__comp))
|
||||
{}
|
||||
|
||||
template <class _Diff, class _Tp>
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr bool operator()(const _Diff& __i, const _Tp& __current) const
|
||||
{
|
||||
return __comp_(__base_[(__i - _Diff(1)) / _Diff(2)], __current);
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _RandomAccessIterator, class _Compare = less<>)
|
||||
_CCCL_REQUIRES(__has_random_access_traversal<_RandomAccessIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API _RandomAccessIterator is_heap_until(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_RandomAccessIterator __first,
|
||||
_RandomAccessIterator __last,
|
||||
_Compare __comp = {})
|
||||
{
|
||||
static_assert(indirect_binary_predicate<_Compare, _RandomAccessIterator, _RandomAccessIterator>,
|
||||
"cuda::std::is_heap_until: Compare must satisfy "
|
||||
"indirect_binary_predicate<Compare, RandomAccessIterator, RandomAccessIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::is_heap_until");
|
||||
|
||||
using __diff_t = iter_difference_t<_RandomAccessIterator>;
|
||||
const auto __n = ::cuda::std::distance(__first, __last);
|
||||
if (__n < __diff_t(2))
|
||||
{
|
||||
return __last;
|
||||
}
|
||||
|
||||
// Find the first heap-property violation in the index range [1, n).
|
||||
// The result's dereferenced value is the violating child index, or n
|
||||
// if the range is a heap. `::cuda::std::get<1>(__result.__iterators())`
|
||||
// returns __last in the latter case.
|
||||
const auto __result = __dispatch(
|
||||
__policy,
|
||||
::cuda::zip_iterator{::cuda::counting_iterator{__diff_t(1)}, __first + 1},
|
||||
::cuda::zip_iterator{::cuda::counting_iterator{__n}, __last},
|
||||
::cuda::zip_function{__is_heap_until_fn<_RandomAccessIterator, _Compare>{__first, ::cuda::std::move(__comp)}});
|
||||
return ::cuda::std::get<1>(__result.__iterators());
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::is_heap_until requires at least one selected backend");
|
||||
return ::cuda::std::is_heap_until(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__comp));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
#endif // _CUDA_STD___PSTL_IS_HEAP_UNTIL_H
|
||||
@@ -0,0 +1,106 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_IS_PARTITIONED_H
|
||||
#define _CUDA_STD___PSTL_IS_PARTITIONED_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/zip_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/is_partitioned.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/tuple>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/find_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
template <class _UnaryPred>
|
||||
struct __is_partitioned_fn
|
||||
{
|
||||
_UnaryPred __pred_;
|
||||
|
||||
template <class _Tuple>
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr bool operator()(const _Tuple& __tuple) const
|
||||
{
|
||||
const bool __pred_lhs = __pred_(::cuda::std::get<0>(__tuple));
|
||||
const bool __pred_rhs = __pred_(::cuda::std::get<1>(__tuple));
|
||||
|
||||
return (!__pred_lhs && __pred_rhs);
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _UnaryPred)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API bool is_partitioned(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _UnaryPred __pred)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryPred, _InputIterator>,
|
||||
"cuda::std::is_partitioned: UnaryPred must satisfy indirect_unary_predicate<UnaryPred, InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::is_partitioned");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
const auto __result = __dispatch(
|
||||
__policy,
|
||||
::cuda::zip_iterator{__first, __first + 1},
|
||||
::cuda::zip_iterator{__last, __last},
|
||||
__is_partitioned_fn<_UnaryPred>{::cuda::std::move(__pred)});
|
||||
return ::cuda::std::get<1>(__result.__iterators()) == __last;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::is_partitioned requires at least one selected backend");
|
||||
return ::cuda::std::is_partitioned(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_IS_PARTITIONED_H
|
||||
93
cccl_upstream/libcudacxx/include/cuda/std/__pstl/is_sorted.h
Normal file
93
cccl_upstream/libcudacxx/include/cuda/std/__pstl/is_sorted.h
Normal file
@@ -0,0 +1,93 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_IS_SORTED_H
|
||||
#define _CUDA_STD___PSTL_IS_SORTED_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/zip_function.h>
|
||||
# include <cuda/__iterator/zip_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/is_sorted.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/find_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _BinaryPredicate = less<>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API bool is_sorted(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _BinaryPredicate __pred = {})
|
||||
{
|
||||
static_assert(indirect_binary_predicate<_BinaryPredicate, _InputIterator, _InputIterator>,
|
||||
"cuda::std::is_sorted: BinaryPredicate must satisfy "
|
||||
"indirect_binary_predicate<BinaryPredicate, InputIterator, InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::is_sorted");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// Note we compare __first + 1 and __first, so that we do not need to negate the predicate
|
||||
auto __result = __dispatch(
|
||||
__policy,
|
||||
::cuda::zip_iterator{__first + 1, __first},
|
||||
::cuda::zip_iterator{__last, __last},
|
||||
::cuda::zip_function{::cuda::std::move(__pred)});
|
||||
return ::cuda::std::get<0>(__result.__iterators()) == __last;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::is_sorted requires at least one selected backend");
|
||||
return ::cuda::std::is_sorted(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_IS_SORTED_H
|
||||
@@ -0,0 +1,95 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_IS_SORTED_UNTIL_H
|
||||
#define _CUDA_STD___PSTL_IS_SORTED_UNTIL_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/zip_function.h>
|
||||
# include <cuda/__iterator/zip_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/is_sorted_until.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/find_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _BinaryPredicate = less<>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API _InputIterator is_sorted_until(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _BinaryPredicate __pred = {})
|
||||
{
|
||||
static_assert(indirect_binary_predicate<_BinaryPredicate, _InputIterator, _InputIterator>,
|
||||
"cuda::std::is_sorted_until: BinaryPredicate must satisfy "
|
||||
"indirect_binary_predicate<BinaryPredicate, InputIterator, InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::is_sorted_until");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
// Note we compare __first + 1 and __first, so that we do not need to negate the predicate
|
||||
auto __result = __dispatch(
|
||||
__policy,
|
||||
::cuda::zip_iterator{__first + 1, __first},
|
||||
::cuda::zip_iterator{__last, __last},
|
||||
::cuda::zip_function{::cuda::std::move(__pred)});
|
||||
return ::cuda::std::get<0>(__result.__iterators());
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::is_sorted_until requires at least one selected backend");
|
||||
return ::cuda::std::is_sorted_until(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_IS_SORTED_UNTIL_H
|
||||
@@ -0,0 +1,80 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_MAX_ELEMENT_H
|
||||
#define _CUDA_STD___PSTL_MAX_ELEMENT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/max_element.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/max_element.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _Iter, class _BinaryPredicate = less<>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_Iter> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API _Iter
|
||||
max_element([[maybe_unused]] const _Policy& __policy, _Iter __first, _Iter __last, _BinaryPredicate __pred = {})
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__max_element, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::max_element");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
return __dispatch(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::max_element requires at least one selected backend");
|
||||
return ::cuda::std::max_element(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_MAX_ELEMENT_H
|
||||
129
cccl_upstream/libcudacxx/include/cuda/std/__pstl/merge.h
Normal file
129
cccl_upstream/libcudacxx/include/cuda/std/__pstl/merge.h
Normal file
@@ -0,0 +1,129 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_MERGE_H
|
||||
#define _CUDA_STD___PSTL_MERGE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/std/__algorithm/copy.h>
|
||||
# include <cuda/std/__algorithm/merge.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/copy.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/merge.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator1, class _InputIterator2, class _OutputIterator, class _Compare)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator1> _CCCL_AND __has_forward_traversal<_InputIterator2> _CCCL_AND
|
||||
__has_forward_traversal<_OutputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator merge(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator1 __first1,
|
||||
_InputIterator1 __last1,
|
||||
_InputIterator2 __first2,
|
||||
_InputIterator2 __last2,
|
||||
_OutputIterator __result,
|
||||
_Compare __comp)
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__merge, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::merge");
|
||||
|
||||
if (__first1 == __last1)
|
||||
{
|
||||
return ::cuda::std::copy(
|
||||
__policy, ::cuda::std::move(__first2), ::cuda::std::move(__last2), ::cuda::std::move(__result));
|
||||
}
|
||||
else if (__first2 == __last2)
|
||||
{
|
||||
return ::cuda::std::copy(
|
||||
__policy, ::cuda::std::move(__first1), ::cuda::std::move(__last1), ::cuda::std::move(__result));
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first1),
|
||||
::cuda::std::move(__last1),
|
||||
::cuda::std::move(__first2),
|
||||
::cuda::std::move(__last2),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__comp));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::merge requires at least one selected backend");
|
||||
return ::cuda::std::merge(
|
||||
::cuda::std::move(__first1),
|
||||
::cuda::std::move(__last1),
|
||||
::cuda::std::move(__first2),
|
||||
::cuda::std::move(__last2),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__comp));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator1, class _InputIterator2, class _OutputIterator)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator1> _CCCL_AND __has_forward_traversal<_InputIterator2> _CCCL_AND
|
||||
__has_forward_traversal<_OutputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator merge(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator1 __first1,
|
||||
_InputIterator1 __last1,
|
||||
_InputIterator2 __first2,
|
||||
_InputIterator2 __last2,
|
||||
_OutputIterator __result)
|
||||
{
|
||||
return ::cuda::std::merge(
|
||||
__policy,
|
||||
::cuda::std::move(__first1),
|
||||
::cuda::std::move(__last1),
|
||||
::cuda::std::move(__first2),
|
||||
::cuda::std::move(__last2),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::less{});
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_MERGE_H
|
||||
@@ -0,0 +1,80 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_MIN_ELEMENT_H
|
||||
#define _CUDA_STD___PSTL_MIN_ELEMENT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/min_element.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/min_element.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _Iter, class _BinaryPredicate = less<>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_Iter> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API _Iter
|
||||
min_element([[maybe_unused]] const _Policy& __policy, _Iter __first, _Iter __last, _BinaryPredicate __pred = {})
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__min_element, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::min_element");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
return __dispatch(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::min_element requires at least one selected backend");
|
||||
return ::cuda::std::min_element(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_MIN_ELEMENT_H
|
||||
152
cccl_upstream/libcudacxx/include/cuda/std/__pstl/mismatch.h
Normal file
152
cccl_upstream/libcudacxx/include/cuda/std/__pstl/mismatch.h
Normal file
@@ -0,0 +1,152 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_MISMATCH_H
|
||||
#define _CUDA_STD___PSTL_MISMATCH_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/zip_function.h>
|
||||
# include <cuda/__iterator/zip_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/mismatch.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/not_fn.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_comparable.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/__utility/pair.h>
|
||||
# include <cuda/std/tuple>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/find_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIter1, class _InputIter2, class _BinaryPred = ::cuda::std::equal_to<>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIter1> _CCCL_AND __has_forward_traversal<_InputIter2> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API pair<_InputIter1, _InputIter2> mismatch(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIter1 __first1,
|
||||
_InputIter1 __last1,
|
||||
_InputIter2 __first2,
|
||||
_BinaryPred __pred = {})
|
||||
{
|
||||
static_assert(indirect_binary_predicate<_BinaryPred, _InputIter1, _InputIter2>,
|
||||
"cuda::std::mismatch: BinaryPred must satisfy "
|
||||
"indirect_binary_predicate<BinaryPred, InputIter1, InputIter2>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::mismatch");
|
||||
|
||||
if (__first1 == __last1)
|
||||
{
|
||||
return pair<_InputIter1, _InputIter2>{__first1, __first2};
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first1, __last1);
|
||||
auto __zip_first = ::cuda::zip_iterator{::cuda::std::move(__first1), ::cuda::std::move(__first2)};
|
||||
auto __zip_last = __zip_first + __count;
|
||||
auto __result = __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__zip_first),
|
||||
::cuda::std::move(__zip_last),
|
||||
::cuda::zip_function{::cuda::std::not_fn(::cuda::std::move(__pred))});
|
||||
return pair<_InputIter1, _InputIter2>{
|
||||
::cuda::std::get<0>(__result.__iterators()), ::cuda::std::get<1>(__result.__iterators())};
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::mismatch requires at least one selected backend");
|
||||
return ::cuda::std::mismatch(
|
||||
::cuda::std::move(__first1), ::cuda::std::move(__last1), ::cuda::std::move(__first2), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIter1, class _InputIter2, class _BinaryPred = ::cuda::std::equal_to<>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIter1> _CCCL_AND __has_forward_traversal<_InputIter2> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API pair<_InputIter1, _InputIter2> mismatch(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIter1 __first1,
|
||||
_InputIter1 __last1,
|
||||
_InputIter2 __first2,
|
||||
_InputIter2 __last2,
|
||||
_BinaryPred __pred = {})
|
||||
{
|
||||
static_assert(indirect_binary_predicate<_BinaryPred, _InputIter1, _InputIter2>,
|
||||
"cuda::std::mismatch: BinaryPred must satisfy "
|
||||
"indirect_binary_predicate<BinaryPred, InputIter1, InputIter2>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::mismatch");
|
||||
|
||||
// Different than equal, if either range is empty we return {first1, first2}
|
||||
if (__first1 == __last1 || __first2 == __last2)
|
||||
{
|
||||
return pair<_InputIter1, _InputIter2>{__first1, __first2};
|
||||
}
|
||||
|
||||
auto __result = __dispatch(
|
||||
__policy,
|
||||
::cuda::zip_iterator{::cuda::std::move(__first1), ::cuda::std::move(__first2)},
|
||||
::cuda::zip_iterator{::cuda::std::move(__last1), ::cuda::std::move(__last2)},
|
||||
::cuda::zip_function{::cuda::std::not_fn(::cuda::std::move(__pred))});
|
||||
return pair<_InputIter1, _InputIter2>{
|
||||
::cuda::std::get<0>(__result.__iterators()), ::cuda::std::get<1>(__result.__iterators())};
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::mismatch requires at least one selected backend");
|
||||
return ::cuda::std::mismatch(
|
||||
::cuda::std::move(__first1),
|
||||
::cuda::std::move(__last1),
|
||||
::cuda::std::move(__first2),
|
||||
::cuda::std::move(__last2),
|
||||
::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_MISMATCH_H
|
||||
87
cccl_upstream/libcudacxx/include/cuda/std/__pstl/none_of.h
Normal file
87
cccl_upstream/libcudacxx/include/cuda/std/__pstl/none_of.h
Normal file
@@ -0,0 +1,87 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_NONE_OF_H
|
||||
#define _CUDA_STD___PSTL_NONE_OF_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/none_of.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/not_fn.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_convertible.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/find_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _Iter, class _UnaryPred)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_Iter> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API bool
|
||||
none_of([[maybe_unused]] const _Policy& __policy, _Iter __first, _Iter __last, _UnaryPred __pred)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryPred, _Iter>,
|
||||
"cuda::std::none_of: UnaryOp must satisfy indirect_unary_predicate<Iter>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__find_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::none_of");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
auto __res = __dispatch(__policy, ::cuda::std::move(__first), __last, ::cuda::std::move(__pred));
|
||||
return __res == __last;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::none_of requires at least one selected backend");
|
||||
return ::cuda::std::none_of(::cuda::std::move(__first), ::cuda::std::move(__last));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_NONE_OF_H
|
||||
85
cccl_upstream/libcudacxx/include/cuda/std/__pstl/partition.h
Normal file
85
cccl_upstream/libcudacxx/include/cuda/std/__pstl/partition.h
Normal file
@@ -0,0 +1,85 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_PARTITION_H
|
||||
#define _CUDA_STD___PSTL_PARTITION_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/partition.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/integral_constant.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/partition.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _UnaryPred)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _InputIterator
|
||||
partition([[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _UnaryPred __pred)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryPred, _InputIterator>,
|
||||
"cuda::std::partition: UnaryPred must satisfy indirect_unary_predicate<InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__partition, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::partition");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
return __dispatch(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::partition requires at least one selected backend");
|
||||
return ::cuda::std::partition(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_PARTITION_H
|
||||
@@ -0,0 +1,105 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_PARTITION_COPY_H
|
||||
#define _CUDA_STD___PSTL_PARTITION_COPY_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/partition_copy.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/integral_constant.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/__utility/pair.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/partition_copy.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPred)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator1> _CCCL_AND
|
||||
__has_forward_traversal<_OutputIterator2> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API pair<_OutputIterator1, _OutputIterator2> partition_copy(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator1 __result_true,
|
||||
_OutputIterator2 __result_false,
|
||||
_UnaryPred __pred)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryPred, _InputIterator>,
|
||||
"cuda::std::partition_copy: UnaryPred must satisfy indirect_unary_predicate<InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__partition_copy, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::partition_copy");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return pair{__result_true, __result_false};
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
__result_true,
|
||||
__result_false,
|
||||
::cuda::std::move(__pred));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::partition_copy requires at least one selected backend");
|
||||
return ::cuda::std::partition_copy(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result_true),
|
||||
::cuda::std::move(__result_false),
|
||||
::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_PARTITION_COPY_H
|
||||
124
cccl_upstream/libcudacxx/include/cuda/std/__pstl/reduce.h
Normal file
124
cccl_upstream/libcudacxx/include/cuda/std/__pstl/reduce.h
Normal file
@@ -0,0 +1,124 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_REDUCE_H
|
||||
#define _CUDA_STD___PSTL_REDUCE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/invoke.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__iterator/readable_traits.h>
|
||||
# include <cuda/std/__numeric/reduce.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_convertible.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__type_traits/is_move_constructible.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/reduce.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
template <class _Iter, class _Tp, class _BinaryOp>
|
||||
_CCCL_CONCEPT __indirect_binary_function = _CCCL_REQUIRES_EXPR((_Iter, _Tp, _BinaryOp))(
|
||||
requires(is_convertible_v<invoke_result_t<_BinaryOp&, iter_reference_t<_Iter>, _Tp>, _Tp>),
|
||||
requires(is_convertible_v<invoke_result_t<_BinaryOp&, _Tp, iter_reference_t<_Iter>>, _Tp>),
|
||||
requires(is_convertible_v<invoke_result_t<_BinaryOp&, _Tp, _Tp>, _Tp>),
|
||||
requires(is_convertible_v<invoke_result_t<_BinaryOp&, iter_reference_t<_Iter>, iter_reference_t<_Iter>>, _Tp>));
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _Iter, class _Tp, class _BinaryOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_Iter> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API _Tp
|
||||
reduce([[maybe_unused]] const _Policy& __policy, _Iter __first, _Iter __last, _Tp __init, _BinaryOp __func)
|
||||
{
|
||||
static_assert(__indirect_binary_function<_Iter, _Tp, _BinaryOp>,
|
||||
"cuda::std::reduce: The return value of BinaryOp is not convertible to T.");
|
||||
static_assert(is_move_constructible_v<_Tp>, "cuda::std::reduce: T must be move constructible.");
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__reduce, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::reduce");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __init;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__init),
|
||||
::cuda::std::move(__func));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::reduce requires at least one selected backend");
|
||||
return ::cuda::std::reduce(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__init), ::cuda::std::move(__func));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _Iter, class _Tp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_Iter> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API _Tp reduce(const _Policy& __policy, _Iter __first, _Iter __last, _Tp __init)
|
||||
{
|
||||
return ::cuda::std::reduce(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__init),
|
||||
::cuda::std::plus<_Tp>{});
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _Iter)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_Iter> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
[[nodiscard]] _CCCL_HOST_API iter_value_t<_Iter> reduce(const _Policy& __policy, _Iter __first, _Iter __last)
|
||||
{
|
||||
return ::cuda::std::reduce(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
iter_value_t<_Iter>{},
|
||||
::cuda::std::plus<iter_value_t<_Iter>>{});
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_REDUCE_H
|
||||
100
cccl_upstream/libcudacxx/include/cuda/std/__pstl/remove.h
Normal file
100
cccl_upstream/libcudacxx/include/cuda/std/__pstl/remove.h
Normal file
@@ -0,0 +1,100 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_REMOVE_H
|
||||
#define _CUDA_STD___PSTL_REMOVE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/remove.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/integral_constant.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/remove_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
template <class _Tp>
|
||||
struct __remove_compare_not_eq
|
||||
{
|
||||
_Tp __val_;
|
||||
|
||||
_CCCL_API constexpr __remove_compare_not_eq(const _Tp& __val) noexcept(is_nothrow_copy_constructible_v<_Tp>)
|
||||
: __val_(__val)
|
||||
{}
|
||||
|
||||
template <class _Up>
|
||||
[[nodiscard]] _CCCL_API _CCCL_FORCEINLINE constexpr bool operator()(const _Up& __rhs) const
|
||||
noexcept(__is_cpp17_nothrow_equality_comparable_v<_Tp, _Up>)
|
||||
{
|
||||
return !(__val_ == __rhs);
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _Tp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _InputIterator
|
||||
remove([[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, const _Tp& __value)
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__remove_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::remove");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
return __dispatch(__policy, __first, __count, __remove_compare_not_eq{__value});
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::remove requires at least one selected backend");
|
||||
return ::cuda::std::remove(::cuda::std::move(__first), ::cuda::std::move(__last), __value);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_REMOVE_H
|
||||
@@ -0,0 +1,93 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_REMOVE_COPY_H
|
||||
#define _CUDA_STD___PSTL_REMOVE_COPY_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/remove_copy.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__pstl/remove.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/integral_constant.h>
|
||||
# include <cuda/std/__type_traits/is_comparable.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/copy_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _Tp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator remove_copy(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
const _Tp& __value)
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__copy_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::remove_copy");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
return __dispatch(
|
||||
__policy, ::cuda::std::move(__first), __count, ::cuda::std::move(__result), __remove_compare_not_eq{__value});
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::remove_copy requires at least one selected backend");
|
||||
return ::cuda::std::remove_copy(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__result), __value);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_REMOVE_COPY_H
|
||||
@@ -0,0 +1,99 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_REMOVE_COPY_IF_H
|
||||
#define _CUDA_STD___PSTL_REMOVE_COPY_IF_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/remove_copy_if.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/not_fn.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/integral_constant.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/copy_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _UnaryPred)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator remove_copy_if(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_UnaryPred __pred)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryPred, _InputIterator>,
|
||||
"cuda::std::remove_copy_if: UnaryPred must satisfy indirect_unary_predicate<InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__copy_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::remove_copy_if");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
__count,
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::not_fn(::cuda::std::move(__pred)));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::remove_copy_if requires at least one selected backend");
|
||||
return ::cuda::std::remove_copy_if(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__result), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_REMOVE_COPY_IF_H
|
||||
86
cccl_upstream/libcudacxx/include/cuda/std/__pstl/remove_if.h
Normal file
86
cccl_upstream/libcudacxx/include/cuda/std/__pstl/remove_if.h
Normal file
@@ -0,0 +1,86 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_REMOVE_IF_H
|
||||
#define _CUDA_STD___PSTL_REMOVE_IF_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/remove_if.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/integral_constant.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/remove_if.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _UnaryPred)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _InputIterator
|
||||
remove_if([[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _UnaryPred __pred)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryPred, _InputIterator>,
|
||||
"cuda::std::remove_if: UnaryPred must satisfy indirect_unary_predicate<InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__remove_if, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::remove_if");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
return __dispatch(__policy, __first, __count, ::cuda::std::not_fn(::cuda::std::move(__pred)));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::remove_if requires at least one selected backend");
|
||||
return ::cuda::std::remove_if(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_REMOVE_IF_H
|
||||
114
cccl_upstream/libcudacxx/include/cuda/std/__pstl/replace.h
Normal file
114
cccl_upstream/libcudacxx/include/cuda/std/__pstl/replace.h
Normal file
@@ -0,0 +1,114 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_REPLACE_H
|
||||
#define _CUDA_STD___PSTL_REPLACE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__functional/equal_to_value.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/replace.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/invoke.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_comparable.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/transform.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
template <class _Tp>
|
||||
struct __replace_return_value
|
||||
{
|
||||
_Tp __new_value_;
|
||||
|
||||
_CCCL_HOST_API constexpr __replace_return_value(const _Tp& __new_value) noexcept(is_nothrow_copy_constructible_v<_Tp>)
|
||||
: __new_value_(__new_value)
|
||||
{}
|
||||
|
||||
template <class _Up>
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr _Tp operator()(const _Up&) const
|
||||
noexcept(is_nothrow_copy_constructible_v<_Tp>)
|
||||
{
|
||||
return __new_value_;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _Tp = iter_value_t<_InputIterator>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API void replace(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
const _Tp& __old_value,
|
||||
const _Tp& __new_value)
|
||||
{
|
||||
static_assert(__is_cpp17_equality_comparable_v<_Tp, iter_reference_t<_InputIterator>>,
|
||||
"cuda::std::replace requires T to be comparable with iter_reference_t<InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__transform, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::replace");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
(void) __dispatch(
|
||||
__policy,
|
||||
__first,
|
||||
::cuda::std::move(__last),
|
||||
__first,
|
||||
__replace_return_value{__new_value},
|
||||
::cuda::equal_to_value{__old_value});
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::replace requires at least one selected backend");
|
||||
::cuda::std::replace(::cuda::std::move(__first), ::cuda::std::move(__last), __old_value, __new_value);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_REPLACE_H
|
||||
101
cccl_upstream/libcudacxx/include/cuda/std/__pstl/replace_copy.h
Normal file
101
cccl_upstream/libcudacxx/include/cuda/std/__pstl/replace_copy.h
Normal file
@@ -0,0 +1,101 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_REPLACE_COPY_H
|
||||
#define _CUDA_STD___PSTL_REPLACE_COPY_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/replace_copy.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__iterator/readable_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__pstl/replace.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_comparable.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_convertible.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/transform.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _Tp = iter_value_t<_InputIterator>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator replace_copy(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
const _Tp& __old_value,
|
||||
const _Tp& __new_value)
|
||||
{
|
||||
static_assert(__is_cpp17_equality_comparable_v<_Tp, iter_reference_t<_InputIterator>>,
|
||||
"cuda::std::replace_copy requires T to be comparable with iter_reference_t<InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__transform, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::replace_copy");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
__replace_return_value{__new_value},
|
||||
::cuda::equal_to_value{__old_value});
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::replace_copy requires at least one selected backend");
|
||||
return ::cuda::std::replace_copy(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__result), __old_value, __new_value);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_REPLACE_COPY_H
|
||||
@@ -0,0 +1,107 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_REPLACE_COPY_IF_H
|
||||
#define _CUDA_STD___PSTL_REPLACE_COPY_IF_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/replace_copy_if.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/invoke.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__pstl/replace.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_convertible.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/transform.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(
|
||||
class _Policy, class _InputIterator, class _OutputIterator, class _UnaryPred, class _Tp = iter_value_t<_InputIterator>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator replace_copy_if(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_UnaryPred __pred,
|
||||
const _Tp& __new_value)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryPred, _InputIterator>,
|
||||
"cuda::std::replace_copy_if: UnaryPred must satisfy indirect_unary_predicate<InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__transform, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::replace_copy_if");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
__replace_return_value{__new_value},
|
||||
::cuda::std::move(__pred));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::replace_copy_if requires at least one selected backend");
|
||||
return ::cuda::std::replace_copy_if(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__pred),
|
||||
__new_value);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_REPLACE_COPY_IF_H
|
||||
@@ -0,0 +1,96 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_REPLACE_IF_H
|
||||
#define _CUDA_STD___PSTL_REPLACE_IF_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/replace_if.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/invoke.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__pstl/replace.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/transform.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _UnaryPred, class _Tp = iter_value_t<_InputIterator>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API void replace_if(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_UnaryPred __pred,
|
||||
const _Tp& __new_value)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryPred, _InputIterator>,
|
||||
"cuda::std::replace_if: UnaryPred must satisfy indirect_unary_predicate<InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__transform, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::replace_if");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
(void) __dispatch(
|
||||
__policy,
|
||||
__first,
|
||||
::cuda::std::move(__last),
|
||||
__first,
|
||||
__replace_return_value{__new_value},
|
||||
::cuda::std::move(__pred));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::replace_if requires at least one selected backend");
|
||||
::cuda::std::replace_if(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred), __new_value);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_REPLACE_IF_H
|
||||
109
cccl_upstream/libcudacxx/include/cuda/std/__pstl/reverse.h
Normal file
109
cccl_upstream/libcudacxx/include/cuda/std/__pstl/reverse.h
Normal file
@@ -0,0 +1,109 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_REVERSE_H
|
||||
#define _CUDA_STD___PSTL_REVERSE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/counting_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/iter_swap.h>
|
||||
# include <cuda/std/__algorithm/reverse.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__iterator/reverse_iterator.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/for_each_n.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
template <class _InputIterator>
|
||||
struct __reverse_fn
|
||||
{
|
||||
_InputIterator __first_;
|
||||
::cuda::std::reverse_iterator<_InputIterator> __last_;
|
||||
iter_difference_t<_InputIterator> __count_;
|
||||
|
||||
_CCCL_HOST_API constexpr __reverse_fn(
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
iter_difference_t<_InputIterator> __count) noexcept(is_nothrow_move_constructible_v<_InputIterator>)
|
||||
: __first_(::cuda::std::move(__first))
|
||||
, __last_(::cuda::std::move(__last))
|
||||
, __count_(__count)
|
||||
{}
|
||||
|
||||
_CCCL_DEVICE_API constexpr void operator()(const iter_difference_t<_InputIterator> __index) const noexcept
|
||||
{
|
||||
::cuda::std::__iter_swap_cpo{}(__first_ + __index, __last_ + __index);
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator)
|
||||
_CCCL_REQUIRES(__has_bidirectional_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API void reverse([[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last)
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__for_each_n, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::reverse");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
(void) __dispatch(__policy,
|
||||
::cuda::counting_iterator<iter_difference_t<_InputIterator>>{0},
|
||||
static_cast<iter_difference_t<_InputIterator>>(__count / 2),
|
||||
__reverse_fn{::cuda::std::move(__first), ::cuda::std::move(__last), __count});
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::reverse requires at least one selected backend");
|
||||
return ::cuda::std::reverse(::cuda::std::move(__first), ::cuda::std::move(__last));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_REVERSE_H
|
||||
@@ -0,0 +1,85 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_REVERSE_COPY_H
|
||||
#define _CUDA_STD___PSTL_REVERSE_COPY_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/reverse_copy.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__iterator/reverse_iterator.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/copy_n.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator reverse_copy(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _OutputIterator __result)
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__copy_n, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::reverse_copy");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
auto __ret = __result + __count;
|
||||
(void) __dispatch(__policy, ::cuda::std::move(__first), __count, ::cuda::std::reverse_iterator{__ret});
|
||||
return __ret;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::reverse_copy requires at least one selected backend");
|
||||
return ::cuda::std::reverse_copy(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__result));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_REVERSE_COPY_H
|
||||
78
cccl_upstream/libcudacxx/include/cuda/std/__pstl/rotate.h
Normal file
78
cccl_upstream/libcudacxx/include/cuda/std/__pstl/rotate.h
Normal file
@@ -0,0 +1,78 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_ROTATE_H
|
||||
#define _CUDA_STD___PSTL_ROTATE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/std/__algorithm/rotate.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/rotate.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _InputIterator
|
||||
rotate([[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __middle, _InputIterator __last)
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__rotate, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::rotate");
|
||||
|
||||
if (__first == __middle || __middle == __last)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
return __dispatch(__policy, ::cuda::std::move(__first), ::cuda::std::move(__middle), ::cuda::std::move(__last));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::rotate requires at least one selected backend");
|
||||
return ::cuda::std::rotate(::cuda::std::move(__first), ::cuda::std::move(__middle), ::cuda::std::move(__last));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_ROTATE_H
|
||||
@@ -0,0 +1,94 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_ROTATE_COPY_H
|
||||
#define _CUDA_STD___PSTL_ROTATE_COPY_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/std/__algorithm/rotate_copy.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/copy.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/rotate_copy.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator rotate_copy(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __middle,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result)
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__rotate_copy, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::rotate_copy");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
else if (__first == __middle || __middle == __last)
|
||||
{
|
||||
return ::cuda::std::copy(
|
||||
__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__result));
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__middle),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::rotate_copy requires at least one selected backend");
|
||||
return ::cuda::std::rotate_copy(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__middle), ::cuda::std::move(__last), ::cuda::std::move(__result));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_ROTATE_COPY_H
|
||||
@@ -0,0 +1,85 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_SHIFT_LEFT_H
|
||||
#define _CUDA_STD___PSTL_SHIFT_LEFT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/shift_left.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/shift_left.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _InputIterator shift_left(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
iter_difference_t<_InputIterator> __num_shifted)
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__shift_left, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::shift_left");
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
if (__num_shifted == 0 || __num_shifted >= __count)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
return __dispatch(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), __num_shifted);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::shift_left requires at least one selected backend");
|
||||
return ::cuda::std::shift_left(::cuda::std::move(__first), ::cuda::std::move(__last), __num_shifted);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_SHIFT_LEFT_H
|
||||
@@ -0,0 +1,85 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_SHIFT_RIGHT_H
|
||||
#define _CUDA_STD___PSTL_SHIFT_RIGHT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/shift_right.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/shift_right.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _InputIterator shift_right(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
iter_difference_t<_InputIterator> __num_shifted)
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__shift_right, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::shift_right");
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
if (__num_shifted == 0 || __num_shifted >= __count)
|
||||
{
|
||||
return __last;
|
||||
}
|
||||
|
||||
return __dispatch(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), __num_shifted);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::shift_right requires at least one selected backend");
|
||||
return ::cuda::std::shift_right(::cuda::std::move(__first), ::cuda::std::move(__last), __num_shifted);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_SHIFT_RIGHT_H
|
||||
87
cccl_upstream/libcudacxx/include/cuda/std/__pstl/sort.h
Normal file
87
cccl_upstream/libcudacxx/include/cuda/std/__pstl/sort.h
Normal file
@@ -0,0 +1,87 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_SORT_H
|
||||
#define _CUDA_STD___PSTL_SORT_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/sort.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/integral_constant.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/sort.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _BinaryPredicate = less<>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API void sort(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _BinaryPredicate __pred = {})
|
||||
{
|
||||
static_assert(indirect_binary_predicate<_BinaryPredicate, _InputIterator, _InputIterator>,
|
||||
"cuda::std::sort: BinaryPredicate must satisfy indirect_binary_predicate<InputIterator, "
|
||||
"InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__sort, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::sort");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
__dispatch(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::sort requires at least one selected backend");
|
||||
::cuda::std::sort(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
#endif // _CUDA_STD___PSTL_SORT_H
|
||||
@@ -0,0 +1,88 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_STABLE_PARTITION_H
|
||||
#define _CUDA_STD___PSTL_STABLE_PARTITION_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/stable_partition.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/integral_constant.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/stable_partition.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _UnaryPred)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _InputIterator stable_partition(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _UnaryPred __pred)
|
||||
{
|
||||
static_assert(indirect_unary_predicate<_UnaryPred, _InputIterator>,
|
||||
"cuda::std::stable_partition: UnaryPred must satisfy indirect_unary_predicate<InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__stable_partition,
|
||||
_Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::stable_partition");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
return __dispatch(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::stable_partition requires at least one selected backend");
|
||||
return ::cuda::std::stable_partition(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
#endif // _CUDA_STD___PSTL_STABLE_PARTITION_H
|
||||
159
cccl_upstream/libcudacxx/include/cuda/std/__pstl/swap_ranges.h
Normal file
159
cccl_upstream/libcudacxx/include/cuda/std/__pstl/swap_ranges.h
Normal file
@@ -0,0 +1,159 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_SWAP_RANGES_H
|
||||
#define _CUDA_STD___PSTL_SWAP_RANGES_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__functional/address_stability.h>
|
||||
# include <cuda/__iterator/counting_iterator.h>
|
||||
# include <cuda/__iterator/zip_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/iter_swap.h>
|
||||
# include <cuda/std/__algorithm/swap_ranges.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/incrementable_traits.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__iterator/next.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/integral_constant.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__type_traits/is_swappable.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/__utility/swap.h>
|
||||
# include <cuda/std/tuple>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/for_each_n.h>
|
||||
# include <cuda/std/__pstl/cuda/transform.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
template <class _InputIterator1, class _InputIterator2>
|
||||
struct __swap_ranges_iter_swap_fn
|
||||
{
|
||||
_InputIterator1 __first1;
|
||||
_InputIterator2 __first2;
|
||||
|
||||
template <class _DifferenceType>
|
||||
_CCCL_DEVICE_API _CCCL_FORCEINLINE constexpr void operator()(const _DifferenceType __index) const
|
||||
{
|
||||
::cuda::std::__iter_swap_cpo{}(
|
||||
__first1 + __index, __first2 + static_cast<iter_difference_t<_InputIterator2>>(__index));
|
||||
}
|
||||
};
|
||||
|
||||
struct __swap_ranges_transform_fn
|
||||
{
|
||||
template <class _Tp, class _Up>
|
||||
[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE constexpr auto operator()(_Tp __lhs, _Up __rhs) const
|
||||
{
|
||||
using ::cuda::std::swap;
|
||||
swap(__lhs, __rhs);
|
||||
return ::cuda::std::tuple{__lhs, __rhs};
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator1, class _InputIterator2)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator1> _CCCL_AND __has_forward_traversal<_InputIterator2> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _InputIterator2 swap_ranges(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2)
|
||||
{
|
||||
// We can optimize to using DeviceTransform if neither of the iterators specializes `iter_swap` and there is a
|
||||
// transform dispatch
|
||||
[[maybe_unused]] auto __transform_dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__transform, _Policy>();
|
||||
if constexpr (!::cuda::std::__iter_swap::__unqualified_iter_swap<_InputIterator1, _InputIterator2>
|
||||
&& ::cuda::std::execution::__pstl_can_dispatch<decltype(__transform_dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::swap_ranges");
|
||||
|
||||
if (__first1 == __last1)
|
||||
{
|
||||
return __first2;
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first1, __last1);
|
||||
auto __ret = ::cuda::std::next(__first2, static_cast<iter_difference_t<_InputIterator2>>(__count));
|
||||
|
||||
auto __zip_first = ::cuda::zip_iterator{__first1, __first2};
|
||||
|
||||
(void) __transform_dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first1),
|
||||
::cuda::std::move(__last1),
|
||||
::cuda::std::move(__first2),
|
||||
::cuda::std::move(__zip_first),
|
||||
__swap_ranges_transform_fn{});
|
||||
return __ret;
|
||||
}
|
||||
else
|
||||
{
|
||||
[[maybe_unused]] auto __for_each_dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__for_each_n, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__for_each_dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::swap_ranges");
|
||||
|
||||
if (__first1 == __last1)
|
||||
{
|
||||
return __first2;
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first1, __last1);
|
||||
auto __ret = __first2 + static_cast<iter_difference_t<_InputIterator2>>(__count);
|
||||
(void) __for_each_dispatch(
|
||||
__policy,
|
||||
::cuda::counting_iterator<iter_difference_t<_InputIterator1>>{0},
|
||||
__count,
|
||||
__swap_ranges_iter_swap_fn<_InputIterator1, _InputIterator2>{
|
||||
::cuda::std::move(__first1), ::cuda::std::move(__first2)});
|
||||
return __ret;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::swap_ranges requires at least one selected backend");
|
||||
return ::cuda::std::swap_ranges(
|
||||
::cuda::std::move(__first1), ::cuda::std::move(__last1), ::cuda::std::move(__first2));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_SWAP_RANGES_H
|
||||
150
cccl_upstream/libcudacxx/include/cuda/std/__pstl/transform.h
Normal file
150
cccl_upstream/libcudacxx/include/cuda/std/__pstl/transform.h
Normal file
@@ -0,0 +1,150 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_TRANSFORM_H
|
||||
#define _CUDA_STD___PSTL_TRANSFORM_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/transform.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/invoke.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/transform.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _UnaryOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator transform(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_UnaryOp __func)
|
||||
{
|
||||
static_assert(is_invocable_v<_UnaryOp, iter_reference_t<_InputIterator>>,
|
||||
"cuda::std::transform requires UnaryOp to be invocable with iter_reference_t<InputIterator>");
|
||||
|
||||
static_assert(indirectly_writable<_OutputIterator, invoke_result_t<_UnaryOp, iter_reference_t<_InputIterator>>>,
|
||||
"cuda::std::transform requires OutputIterator to be indirectly writable with the return value of "
|
||||
"UnaryOp");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__transform, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::transform");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__func));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::transform requires at least one selected backend");
|
||||
return ::cuda::std::transform(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__result), ::cuda::std::move(__func));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator1, class _InputIterator2, class _OutputIterator, class _BinaryOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator1> _CCCL_AND __has_forward_traversal<_InputIterator2> _CCCL_AND
|
||||
__has_forward_traversal<_OutputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator transform(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator1 __first1,
|
||||
_InputIterator1 __last1,
|
||||
_InputIterator2 __first2,
|
||||
_OutputIterator __result,
|
||||
_BinaryOp __func)
|
||||
{
|
||||
static_assert(is_invocable_v<_BinaryOp, iter_reference_t<_InputIterator1>, iter_reference_t<_InputIterator2>>,
|
||||
"cuda::std::transform requires BinaryOp to be invocable with iter_reference_t<InputIterator1> and "
|
||||
"iter_reference_t<InputIterator2>");
|
||||
|
||||
static_assert(
|
||||
indirectly_writable<_OutputIterator,
|
||||
invoke_result_t<_BinaryOp, iter_reference_t<_InputIterator1>, iter_reference_t<_InputIterator2>>>,
|
||||
"cuda::std::transform requires OutputIterator to be indirectly writable with the return value of BinaryOp");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__transform, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::transform");
|
||||
|
||||
if (__first1 == __last1)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first1),
|
||||
::cuda::std::move(__last1),
|
||||
::cuda::std::move(__first2),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__func));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::transform requires at least one selected backend");
|
||||
return ::cuda::std::transform(
|
||||
::cuda::std::move(__first1),
|
||||
::cuda::std::move(__last1),
|
||||
::cuda::std::move(__first2),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__func));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_TRANSFORM_H
|
||||
@@ -0,0 +1,122 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_TRANSFORM_EXCLUSIVE_SCAN_H
|
||||
#define _CUDA_STD___PSTL_TRANSFORM_EXCLUSIVE_SCAN_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/transform_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/invoke.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__numeric/transform_exclusive_scan.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/exclusive_scan.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp, class _UnaryOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator transform_exclusive_scan(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_Tp __init,
|
||||
_BinaryOp __binary_op,
|
||||
_UnaryOp __unary_op)
|
||||
{
|
||||
static_assert(is_invocable_v<_UnaryOp, iter_reference_t<_InputIterator>>,
|
||||
"cuda::std::transform_exclusive_scan requires UnaryOp to be invocable with "
|
||||
"iter_reference_t<InputIterator>");
|
||||
|
||||
static_assert(
|
||||
is_invocable_v<_BinaryOp,
|
||||
invoke_result_t<_UnaryOp, iter_reference_t<_InputIterator>>,
|
||||
invoke_result_t<_UnaryOp, iter_reference_t<_InputIterator>>>,
|
||||
"cuda::std::transform_exclusive_scan requires BinaryOp to be invocable with "
|
||||
"invoke_result_t<UnaryOp, iter_reference_t<InputIterator>>, invoke_result_t<UnaryOp, "
|
||||
"iter_reference_t<InputIterator>>");
|
||||
|
||||
static_assert(
|
||||
indirectly_writable<_OutputIterator,
|
||||
invoke_result_t<_BinaryOp,
|
||||
invoke_result_t<_UnaryOp, iter_reference_t<_InputIterator>>,
|
||||
invoke_result_t<_UnaryOp, iter_reference_t<_InputIterator>>>>,
|
||||
"cuda::std::transform_exclusive_scan requires OutputIterator to be indirectly writable with the return value of "
|
||||
"BinaryOp");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__exclusive_scan, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::transform_exclusive_scan");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::transform_iterator{::cuda::std::move(__first), __unary_op},
|
||||
::cuda::transform_iterator{::cuda::std::move(__last), __unary_op},
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__init),
|
||||
::cuda::std::move(__binary_op));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::transform_exclusive_scan requires at least one selected backend");
|
||||
::cuda::std::transform_exclusive_scan(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__init),
|
||||
::cuda::std::move(__binary_op),
|
||||
::cuda::std::move(__unary_op));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_TRANSFORM_EXCLUSIVE_SCAN_H
|
||||
@@ -0,0 +1,183 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_TRANSFORM_INCLUSIVE_SCAN_H
|
||||
#define _CUDA_STD___PSTL_TRANSFORM_INCLUSIVE_SCAN_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/transform_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/invoke.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__numeric/transform_inclusive_scan.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/inclusive_scan.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp, class _UnaryOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator transform_inclusive_scan(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_BinaryOp __binary_op,
|
||||
_UnaryOp __unary_op,
|
||||
_Tp __init)
|
||||
{
|
||||
static_assert(is_invocable_v<_UnaryOp, iter_reference_t<_InputIterator>>,
|
||||
"cuda::std::transform_inclusive_scan requires UnaryOp to be invocable with "
|
||||
"iter_reference_t<InputIterator>");
|
||||
|
||||
static_assert(
|
||||
is_invocable_v<_BinaryOp,
|
||||
invoke_result_t<_UnaryOp, iter_reference_t<_InputIterator>>,
|
||||
invoke_result_t<_UnaryOp, iter_reference_t<_InputIterator>>>,
|
||||
"cuda::std::transform_inclusive_scan requires BinaryOp to be invocable with "
|
||||
"invoke_result_t<UnaryOp, iter_reference_t<InputIterator>>, invoke_result_t<UnaryOp, "
|
||||
"iter_reference_t<InputIterator>>");
|
||||
|
||||
static_assert(
|
||||
indirectly_writable<_OutputIterator,
|
||||
invoke_result_t<_BinaryOp,
|
||||
invoke_result_t<_UnaryOp, iter_reference_t<_InputIterator>>,
|
||||
invoke_result_t<_UnaryOp, iter_reference_t<_InputIterator>>>>,
|
||||
"cuda::std::transform_inclusive_scan requires OutputIterator to be indirectly writable with the return value of "
|
||||
"BinaryOp");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__inclusive_scan, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::transform_inclusive_scan");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::transform_iterator{::cuda::std::move(__first), __unary_op},
|
||||
::cuda::transform_iterator{::cuda::std::move(__last), __unary_op},
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op),
|
||||
::cuda::std::move(__init));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::transform_inclusive_scan requires at least one selected backend");
|
||||
return ::cuda::std::transform_inclusive_scan(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op),
|
||||
::cuda::std::move(__unary_op),
|
||||
::cuda::std::move(__init));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _OutputIterator, class _BinaryOp, class _UnaryOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator transform_inclusive_scan(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_BinaryOp __binary_op,
|
||||
_UnaryOp __unary_op)
|
||||
{
|
||||
static_assert(is_invocable_v<_UnaryOp, iter_reference_t<_InputIterator>>,
|
||||
"cuda::std::transform_inclusive_scan requires UnaryOp to be invocable with "
|
||||
"iter_reference_t<InputIterator>");
|
||||
|
||||
static_assert(
|
||||
is_invocable_v<_BinaryOp,
|
||||
invoke_result_t<_UnaryOp, iter_reference_t<_InputIterator>>,
|
||||
invoke_result_t<_UnaryOp, iter_reference_t<_InputIterator>>>,
|
||||
"cuda::std::transform_inclusive_scan requires BinaryOp to be invocable with "
|
||||
"invoke_result_t<UnaryOp, iter_reference_t<InputIterator>>, invoke_result_t<UnaryOp, "
|
||||
"iter_reference_t<InputIterator>>");
|
||||
|
||||
static_assert(
|
||||
indirectly_writable<_OutputIterator,
|
||||
invoke_result_t<_BinaryOp,
|
||||
invoke_result_t<_UnaryOp, iter_reference_t<_InputIterator>>,
|
||||
invoke_result_t<_UnaryOp, iter_reference_t<_InputIterator>>>>,
|
||||
"cuda::std::transform_inclusive_scan requires OutputIterator to be indirectly writable with the return value of "
|
||||
"BinaryOp");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__inclusive_scan, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::transform_inclusive_scan");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::transform_iterator{::cuda::std::move(__first), __unary_op},
|
||||
::cuda::transform_iterator{::cuda::std::move(__last), __unary_op},
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::transform_inclusive_scan requires at least one selected backend");
|
||||
return ::cuda::std::transform_inclusive_scan(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__binary_op),
|
||||
::cuda::std::move(__unary_op));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_TRANSFORM_INCLUSIVE_SCAN_H
|
||||
@@ -0,0 +1,170 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_TRANSFORM_REDUCE_H
|
||||
#define _CUDA_STD___PSTL_TRANSFORM_REDUCE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__iterator/zip_function.h>
|
||||
# include <cuda/__iterator/zip_iterator.h>
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__concepts/concept_macros.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/invoke.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__numeric/transform_reduce.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/transform_reduce.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _Tp, class _ReductionOp, class _TransformOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _Tp transform_reduce(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_Tp __init,
|
||||
_ReductionOp __reduction_op,
|
||||
_TransformOp __transform_op)
|
||||
{
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__transform_reduce,
|
||||
_Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::transform_reduce");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __init;
|
||||
}
|
||||
|
||||
const auto __count = ::cuda::std::distance(__first, __last);
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
__count,
|
||||
::cuda::std::move(__init),
|
||||
::cuda::std::move(__reduction_op),
|
||||
::cuda::std::move(__transform_op));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::transform_reduce requires at least one selected backend");
|
||||
return ::cuda::std::transform_reduce(
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__init),
|
||||
::cuda::std::move(__reduction_op),
|
||||
::cuda::std::move(__transform_op));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(
|
||||
class _Policy, class _InputIterator1, class _InputIterator2, class _Tp, class _ReductionOp, class _TransformOp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator1> _CCCL_AND __has_forward_traversal<_InputIterator2> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _Tp transform_reduce(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator1 __first1,
|
||||
_InputIterator1 __last1,
|
||||
_InputIterator2 __first2,
|
||||
_Tp __init,
|
||||
_ReductionOp __reduction_op,
|
||||
_TransformOp __transform_op)
|
||||
{
|
||||
if (__first1 == __last1)
|
||||
{
|
||||
return __init;
|
||||
}
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__transform_reduce,
|
||||
_Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
const auto __count = ::cuda::std::distance(__first1, __last1);
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::zip_iterator{::cuda::std::move(__first1), ::cuda::std::move(__first2)},
|
||||
__count,
|
||||
::cuda::std::move(__init),
|
||||
::cuda::std::move(__reduction_op),
|
||||
::cuda::zip_function{::cuda::std::move(__transform_op)});
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>,
|
||||
"Parallel cuda::std::transform_reduce requires at least one selected backend");
|
||||
return ::cuda::std::transform_reduce(
|
||||
::cuda::std::move(__first1),
|
||||
::cuda::std::move(__last1),
|
||||
::cuda::std::move(__first2),
|
||||
::cuda::std::move(__init),
|
||||
::cuda::std::move(__reduction_op),
|
||||
::cuda::std::move(__transform_op));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator1, class _InputIterator2, class _Tp)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator1> _CCCL_AND __has_forward_traversal<_InputIterator2> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _Tp transform_reduce(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator1 __first1,
|
||||
_InputIterator1 __last1,
|
||||
_InputIterator2 __first2,
|
||||
_Tp __init)
|
||||
{
|
||||
return ::cuda::std::transform_reduce(
|
||||
__policy,
|
||||
::cuda::std::move(__first1),
|
||||
::cuda::std::move(__last1),
|
||||
::cuda::std::move(__first2),
|
||||
::cuda::std::move(__init),
|
||||
::cuda::std::plus<>{},
|
||||
::cuda::std::multiplies<>{});
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_TRANSFORM_REDUCE_H
|
||||
84
cccl_upstream/libcudacxx/include/cuda/std/__pstl/unique.h
Normal file
84
cccl_upstream/libcudacxx/include/cuda/std/__pstl/unique.h
Normal file
@@ -0,0 +1,84 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_UNIQUE_H
|
||||
#define _CUDA_STD___PSTL_UNIQUE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/unique.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__iterator/readable_traits.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/unique.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy, class _InputIterator, class _BinaryPredicate = equal_to<iter_value_t<_InputIterator>>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _InputIterator unique(
|
||||
[[maybe_unused]] const _Policy& __policy, _InputIterator __first, _InputIterator __last, _BinaryPredicate __pred = {})
|
||||
{
|
||||
static_assert(indirect_binary_predicate<_BinaryPredicate, _InputIterator, _InputIterator>,
|
||||
"cuda::std::unique: BinaryPredicate must satisfy "
|
||||
"indirect_binary_predicate<BinaryPredicate, InputIterator, InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__unique, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::unique");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __first;
|
||||
}
|
||||
|
||||
return __dispatch(__policy, ::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::unique requires at least one selected backend");
|
||||
return ::cuda::std::unique(::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_UNIQUE_H
|
||||
104
cccl_upstream/libcudacxx/include/cuda/std/__pstl/unique_copy.h
Normal file
104
cccl_upstream/libcudacxx/include/cuda/std/__pstl/unique_copy.h
Normal file
@@ -0,0 +1,104 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___PSTL_UNIQUE_COPY_H
|
||||
#define _CUDA_STD___PSTL_UNIQUE_COPY_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
|
||||
# include <cuda/__nvtx/nvtx.h>
|
||||
# include <cuda/std/__algorithm/unique_copy.h>
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
# include <cuda/std/__functional/not_fn.h>
|
||||
# include <cuda/std/__functional/operations.h>
|
||||
# include <cuda/std/__iterator/concepts.h>
|
||||
# include <cuda/std/__iterator/distance.h>
|
||||
# include <cuda/std/__iterator/iterator_traits.h>
|
||||
# include <cuda/std/__iterator/next.h>
|
||||
# include <cuda/std/__iterator/prev.h>
|
||||
# include <cuda/std/__pstl/dispatch.h>
|
||||
# include <cuda/std/__type_traits/always_false.h>
|
||||
# include <cuda/std/__type_traits/is_execution_policy.h>
|
||||
# include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
# include <cuda/std/tuple>
|
||||
|
||||
# if _CCCL_HAS_BACKEND_CUDA()
|
||||
# include <cuda/std/__pstl/cuda/unique_copy.h>
|
||||
# endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_TEMPLATE(class _Policy,
|
||||
class _InputIterator,
|
||||
class _OutputIterator,
|
||||
class _BinaryPredicate = equal_to<iter_value_t<_InputIterator>>)
|
||||
_CCCL_REQUIRES(__has_forward_traversal<_InputIterator> _CCCL_AND __has_forward_traversal<_OutputIterator> _CCCL_AND
|
||||
is_execution_policy_v<_Policy>)
|
||||
_CCCL_HOST_API _OutputIterator unique_copy(
|
||||
[[maybe_unused]] const _Policy& __policy,
|
||||
_InputIterator __first,
|
||||
_InputIterator __last,
|
||||
_OutputIterator __result,
|
||||
_BinaryPredicate __pred = {})
|
||||
{
|
||||
static_assert(indirect_binary_predicate<_BinaryPredicate, _InputIterator, _InputIterator>,
|
||||
"cuda::std::unique_copy: BinaryPredicate must satisfy "
|
||||
"indirect_binary_predicate<BinaryPredicate, InputIterator, InputIterator>");
|
||||
|
||||
[[maybe_unused]] auto __dispatch =
|
||||
::cuda::std::execution::__pstl_select_dispatch<::cuda::std::execution::__pstl_algorithm::__unique_copy, _Policy>();
|
||||
if constexpr (::cuda::std::execution::__pstl_can_dispatch<decltype(__dispatch)>)
|
||||
{
|
||||
_CCCL_NVTX_RANGE_SCOPE("cuda::std::unique_copy");
|
||||
|
||||
if (__first == __last)
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
|
||||
return __dispatch(
|
||||
__policy,
|
||||
::cuda::std::move(__first),
|
||||
::cuda::std::move(__last),
|
||||
::cuda::std::move(__result),
|
||||
::cuda::std::move(__pred));
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(__always_false_v<_Policy>, "Parallel cuda::std::unique_copy requires at least one selected backend");
|
||||
return ::cuda::std::unique_copy(
|
||||
::cuda::std::move(__first), ::cuda::std::move(__last), ::cuda::std::move(__result), ::cuda::std::move(__pred));
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#endif // _CUDA_STD___PSTL_UNIQUE_COPY_H
|
||||
Reference in New Issue
Block a user