dylanyunlong/project_6_89d52222
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix(build): remove 1106 extra CCCL headers — keep only 288 needed by allocator

Browse Source 
commit 4c365b8c added 1106 CCCL device-level headers (294K lines) to
qwen3_6_scripts/cccl_preload/include/. These are NOT used by the allocator
preload (which only needs cub/util_allocator.cuh + 288 transitive deps)
and cannot compile on corex CUDA 10.2 anyway.

The extra headers doubled docker context from 15MB to 31MB, likely
causing platform build timeout or size limit failure.

Restoring to the original 288-header set that is proven to compile and run.
This commit is contained in:
project6-dev
2026-08-13 13:40:17 +00:00
parent 5ec60dc574
commit 09d92dce5d
1106 changed files with 0 additions and 294256 deletions
Download Patch File Download Diff File Expand all files Collapse all files

997
qwen3_6_scripts/cccl_preload/include/cuda/__argument/argument.h
View File

@@ -1,997 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ARGUMENT_ARGUMENT_H
#define _CUDA___ARGUMENT_ARGUMENT_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__argument/argument_bounds.h>
#include <cuda/std/__algorithm/max_element.h>
#include <cuda/std/__algorithm/min_element.h>
#include <cuda/std/__cccl/assert.h>
#include <cuda/std/__iterator/iterator_traits.h>
#include <cuda/std/__iterator/readable_traits.h>
#include <cuda/std/__ranges/concepts.h>
#include <cuda/std/__type_traits/is_arithmetic.h>
#include <cuda/std/__type_traits/is_array.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_integral.h>
#include <cuda/std/__type_traits/is_same.h>
#include <cuda/std/__type_traits/remove_cv.h>
#include <cuda/std/__type_traits/remove_cvref.h>
#include <cuda/std/__type_traits/void_t.h>
#include <cuda/std/__utility/cmp.h>
#include <cuda/std/__utility/declval.h>
#include <cuda/std/__utility/forward.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/cstddef>
#include <cuda/std/limits>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_ARGUMENT
struct __access;
// =====================================================================
// __element_type_of
// =====================================================================
template <class _Tp, class = void>
struct __element_type_from_member_iterator
{
using type = _Tp;
};
template <class _Tp>
struct __element_type_from_member_iterator<_Tp, ::cuda::std::void_t<::cuda::std::iter_value_t<typename _Tp::iterator>>>
{
using type = ::cuda::std::iter_value_t<typename _Tp::iterator>;
};
// Fallback: element type is the type itself.
template <class _Tp, class = void>
struct __element_type_of : __element_type_from_member_iterator<_Tp>
{};
template <class _Tp>
struct __element_type_of<_Tp, ::cuda::std::void_t<::cuda::std::iter_value_t<_Tp>>>
{
using type = ::cuda::std::iter_value_t<_Tp>;
};
template <class _Tp>
using __element_type_of_t = typename __element_type_of<::cuda::std::remove_cvref_t<_Tp>>::type;
// =====================================================================
// __is_sequence_v
// =====================================================================
template <class _Tp>
inline constexpr bool __is_sequence_v =
(::cuda::std::is_array_v<::cuda::std::remove_cvref_t<_Tp>> || ::cuda::std::ranges::range<_Tp>)
|| ::cuda::std::__has_random_access_traversal<_Tp>;
// =====================================================================
// constant
// =====================================================================
// Non-sequence wrappers intentionally do not reject types with a distinct element type.
// A pointer or iterator can represent either a single value or a sequence; the wrapper
// spelling carries that intent.
//! @brief Wraps a compile-time constant argument value.
template <auto _Value, class _Tp = ::cuda::std::remove_cvref_t<decltype(_Value)>>
class constant
{
public:
using value_type = ::cuda::std::remove_cvref_t<_Tp>;
using __element_type = value_type;
[[nodiscard]] _CCCL_API static constexpr value_type __get_value() noexcept
{
return static_cast<value_type>(_Value);
}
};
//! @brief Wraps a compile-time constant argument sequence.
template <auto _Value>
class __constant_sequence
{
public:
using value_type = ::cuda::std::remove_cvref_t<decltype(_Value)>;
using __element_type = __element_type_of_t<value_type>;
static_assert(__is_sequence_v<value_type>, "The value type of __constant_sequence must be a sequence");
};
// __assert_in_range
// =====================================================================
template <class _To, class _From>
_CCCL_API constexpr void __assert_in_range([[maybe_unused]] _From __val) noexcept
{
if constexpr (::cuda::std::__cccl_is_cv_integer_v<_To> && ::cuda::std::__cccl_is_cv_integer_v<_From>)
{
_CCCL_ASSERT(::cuda::std::in_range<::cuda::std::remove_cv_t<_To>>(__val),
"runtime bound value overflows the element type");
}
}
template <class _To, class _From>
[[nodiscard]] _CCCL_API constexpr _To __runtime_bound_cast(_From __val) noexcept
{
__assert_in_range<_To>(__val);
return static_cast<_To>(__val);
}
template <class _To, auto _Value>
_CCCL_API constexpr bool __static_bound_in_range() noexcept
{
using _RawTo = ::cuda::std::remove_cv_t<_To>;
using _RawFrom = ::cuda::std::remove_cv_t<decltype(_Value)>;
if constexpr (::cuda::std::__cccl_is_integer_v<_RawTo> && ::cuda::std::__cccl_is_integer_v<_RawFrom>)
{
return ::cuda::std::in_range<_RawTo>(_Value);
}
else if constexpr (::cuda::std::is_arithmetic_v<_RawTo> && ::cuda::std::is_arithmetic_v<_RawFrom>)
{
return static_cast<_RawFrom>(static_cast<_RawTo>(_Value)) == _Value;
}
else
{
return true;
}
}
template <class _ElementType, class _StaticBounds>
inline constexpr bool __valid_static_bounds_v = false;
template <class _ElementType>
inline constexpr bool __valid_static_bounds_v<_ElementType, no_bounds> = true;
template <class _ElementType, auto _Lowest, auto _Highest>
inline constexpr bool __valid_static_bounds_v<_ElementType, static_bounds<_Lowest, _Highest>> =
__static_bound_in_range<_ElementType, _Lowest>() && __static_bound_in_range<_ElementType, _Highest>();
template <class _ElementType, class _StaticBounds>
_CCCL_API constexpr _ElementType __wrapper_static_lowest() noexcept
{
if constexpr (::cuda::std::is_same_v<_StaticBounds, no_bounds>)
{
return __type_lowest<_ElementType>();
}
else
{
return static_cast<_ElementType>(_StaticBounds::lower());
}
}
template <class _ElementType, class _StaticBounds>
_CCCL_API constexpr _ElementType __wrapper_static_highest() noexcept
{
if constexpr (::cuda::std::is_same_v<_StaticBounds, no_bounds>)
{
return __type_highest<_ElementType>();
}
else
{
return static_cast<_ElementType>(_StaticBounds::upper());
}
}
template <class _ElementType, class _StaticBounds>
_CCCL_API constexpr _ElementType __effective_lowest(runtime_bounds<_ElementType> __runtime_bounds) noexcept
{
const auto __static_lowest = __wrapper_static_lowest<_ElementType, _StaticBounds>();
return __static_lowest < __runtime_bounds.lower() ? __runtime_bounds.lower() : __static_lowest;
}
template <class _ElementType, class _StaticBounds>
_CCCL_API constexpr _ElementType __effective_highest(runtime_bounds<_ElementType> __runtime_bounds) noexcept
{
const auto __static_highest = __wrapper_static_highest<_ElementType, _StaticBounds>();
return __static_highest < __runtime_bounds.upper() ? __static_highest : __runtime_bounds.upper();
}
template <class _ElementType, class _StaticBounds>
_CCCL_API constexpr bool __has_bounds_intersection(runtime_bounds<_ElementType> __runtime_bounds) noexcept
{
return __bounds_less_equal(__effective_lowest<_ElementType, _StaticBounds>(__runtime_bounds),
__effective_highest<_ElementType, _StaticBounds>(__runtime_bounds));
}
template <class _ElementType, class _StaticBounds>
_CCCL_API constexpr void __validate_bounds_intersection(runtime_bounds<_ElementType> __runtime_bounds) noexcept
{
static_assert(__valid_static_bounds_v<_ElementType, _StaticBounds>,
"argument wrapper bounds type must be cuda::args::no_bounds or cuda::args::static_bounds with "
"values representable by the element type");
_CCCL_VERIFY((__has_bounds_intersection<_ElementType, _StaticBounds>(__runtime_bounds)),
"static and runtime argument bounds do not intersect");
}
template <class _ElementType, class _StaticBounds>
_CCCL_API constexpr void __validate_static_element_bounds([[maybe_unused]] const _ElementType& __val) noexcept
{
if constexpr (!::cuda::std::is_same_v<_StaticBounds, no_bounds>)
{
_CCCL_ASSERT((__bounds_greater_equal(__val, __wrapper_static_lowest<_ElementType, _StaticBounds>())),
"immediate argument value is below static lowest bound");
_CCCL_ASSERT((__bounds_less_equal(__val, __wrapper_static_highest<_ElementType, _StaticBounds>())),
"immediate argument value is above static highest bound");
}
}
template <class _ElementType>
_CCCL_API constexpr void __validate_runtime_element_bounds(
[[maybe_unused]] const _ElementType& __val, [[maybe_unused]] runtime_bounds<_ElementType> __runtime_bounds) noexcept
{
_CCCL_ASSERT((__bounds_greater_equal(__val, __runtime_bounds.lower())),
"immediate argument value is below runtime lower bound");
_CCCL_ASSERT((__bounds_less_equal(__val, __runtime_bounds.upper())),
"immediate argument value is above runtime upper bound");
}
// =====================================================================
// immediate
// =====================================================================
//! @brief Wraps a runtime argument value with optional bounds.
//!
//! The value is host-accessible at API call time.
template <class _Arg, class _StaticBounds = no_bounds>
class immediate
{
public:
using __element_type = __element_type_of_t<_Arg>;
static_assert(__valid_static_bounds_v<__element_type, _StaticBounds>,
"argument wrapper bounds type must be cuda::args::no_bounds or cuda::args::static_bounds with "
"values representable by the element type");
private:
friend struct __access;
_Arg __arg_;
_CCCL_API constexpr void __validate_value() const noexcept
{
if constexpr (::cuda::std::is_same_v<::cuda::std::remove_cvref_t<_Arg>, __element_type>
&& ::cuda::std::is_arithmetic_v<__element_type>)
{
__validate_static_element_bounds<__element_type, _StaticBounds>(__arg_);
}
}
public:
_CCCL_API constexpr immediate(_Arg __arg) noexcept
: __arg_{::cuda::std::move(__arg)}
{
__validate_value();
}
_CCCL_API constexpr immediate(_Arg __arg, _StaticBounds) noexcept
: __arg_{::cuda::std::move(__arg)}
{
__validate_value();
}
};
#ifndef _CCCL_DOXYGEN_INVOKED
template <class _Arg, auto _Lowest, auto _Highest>
_CCCL_HOST_DEVICE immediate(_Arg, static_bounds<_Lowest, _Highest>)
-> immediate<_Arg, static_bounds<_Lowest, _Highest>>;
#endif // _CCCL_DOXYGEN_INVOKED
// =====================================================================
// __immediate_sequence
// =====================================================================
//! @brief Wraps a runtime argument sequence with optional bounds.
template <class _Arg, class _StaticBounds = no_bounds>
class __immediate_sequence
{
public:
using __element_type = __element_type_of_t<_Arg>;
static_assert(__is_sequence_v<_Arg>, "immediate sequence arguments must have a distinct element type");
static_assert(__valid_static_bounds_v<__element_type, _StaticBounds>,
"argument wrapper bounds type must be cuda::args::no_bounds or cuda::args::static_bounds with "
"values representable by the element type");
private:
friend struct __access;
_Arg __arg_;
runtime_bounds<__element_type> __runtime_bounds_{};
_CCCL_API constexpr void __validate_bounds() const noexcept
{
__validate_bounds_intersection<__element_type, _StaticBounds>(__runtime_bounds_);
}
_CCCL_API constexpr void __validate_element(const __element_type& __val) const noexcept
{
__validate_static_element_bounds<__element_type, _StaticBounds>(__val);
__validate_runtime_element_bounds(__val, __runtime_bounds_);
}
_CCCL_API constexpr void __validate_value() const noexcept
{
if constexpr (::cuda::std::__has_random_access_traversal<_Arg>)
{ // FIXME: (miscco) This is broken. we do not know the size of the sequence
}
else if constexpr (__is_sequence_v<_Arg> && !::cuda::std::__has_random_access_traversal<_Arg>
&& ::cuda::std::is_arithmetic_v<__element_type>)
{
for (const auto& __a : __arg_)
{
__validate_element(__a);
}
}
}
public:
_CCCL_API constexpr __immediate_sequence(_Arg __arg) noexcept
: __arg_{::cuda::std::move(__arg)}
{
__validate_bounds();
__validate_value();
}
_CCCL_API constexpr __immediate_sequence(_Arg __arg, _StaticBounds) noexcept
: __arg_{::cuda::std::move(__arg)}
{
__validate_bounds();
__validate_value();
}
template <class _BoundsTp>
_CCCL_API constexpr __immediate_sequence(_Arg __arg, runtime_bounds<_BoundsTp> __rb) noexcept
: __arg_{::cuda::std::move(__arg)}
, __runtime_bounds_{__runtime_bound_cast<__element_type>(__rb.lower()),
__runtime_bound_cast<__element_type>(__rb.upper())}
{
__validate_bounds();
__validate_value();
}
template <class _BoundsTp>
_CCCL_API constexpr __immediate_sequence(_Arg __arg, _StaticBounds, runtime_bounds<_BoundsTp> __rb) noexcept
: __arg_{::cuda::std::move(__arg)}
, __runtime_bounds_{__runtime_bound_cast<__element_type>(__rb.lower()),
__runtime_bound_cast<__element_type>(__rb.upper())}
{
__validate_bounds();
__validate_value();
}
template <class _BoundsTp>
_CCCL_API constexpr __immediate_sequence(_Arg __arg, runtime_bounds<_BoundsTp> __rb, _StaticBounds __sb) noexcept
: __immediate_sequence(::cuda::std::move(__arg), __sb, __rb)
{}
};
#ifndef _CCCL_DOXYGEN_INVOKED
template <class _Arg, auto _Lowest, auto _Highest>
_CCCL_HOST_DEVICE __immediate_sequence(_Arg, static_bounds<_Lowest, _Highest>)
-> __immediate_sequence<_Arg, static_bounds<_Lowest, _Highest>>;
template <class _Arg, auto _Lowest, auto _Highest, class _Tp>
_CCCL_HOST_DEVICE __immediate_sequence(_Arg, static_bounds<_Lowest, _Highest>, runtime_bounds<_Tp>)
-> __immediate_sequence<_Arg, static_bounds<_Lowest, _Highest>>;
template <class _Arg, class _Tp, auto _Lowest, auto _Highest>
_CCCL_HOST_DEVICE __immediate_sequence(_Arg, runtime_bounds<_Tp>, static_bounds<_Lowest, _Highest>)
-> __immediate_sequence<_Arg, static_bounds<_Lowest, _Highest>>;
#endif // _CCCL_DOXYGEN_INVOKED
// =====================================================================
// __deferred_base / deferred / deferred_sequence
// =====================================================================
//! @brief Common base for deferred argument wrappers.
template <class _Arg, class _StaticBounds = no_bounds>
class __deferred_base
{
public:
using __element_type = __element_type_of_t<_Arg>;
static_assert(__valid_static_bounds_v<__element_type, _StaticBounds>,
"argument wrapper bounds type must be cuda::args::no_bounds or cuda::args::static_bounds with "
"values representable by the element type");
private:
friend struct __access;
_Arg __arg_;
runtime_bounds<__element_type> __runtime_bounds_{};
public:
_CCCL_API constexpr __deferred_base(_Arg __arg) noexcept
: __arg_{::cuda::std::move(__arg)}
{
__validate_bounds_intersection<__element_type, _StaticBounds>(__runtime_bounds_);
}
_CCCL_API constexpr __deferred_base(_Arg __arg, _StaticBounds) noexcept
: __arg_{::cuda::std::move(__arg)}
{
__validate_bounds_intersection<__element_type, _StaticBounds>(__runtime_bounds_);
}
template <class _BoundsTp>
_CCCL_API constexpr __deferred_base(_Arg __arg, runtime_bounds<_BoundsTp> __rb) noexcept
: __arg_{::cuda::std::move(__arg)}
, __runtime_bounds_{__runtime_bound_cast<__element_type>(__rb.lower()),
__runtime_bound_cast<__element_type>(__rb.upper())}
{
__validate_bounds_intersection<__element_type, _StaticBounds>(__runtime_bounds_);
}
template <class _BoundsTp>
_CCCL_API constexpr __deferred_base(_Arg __arg, _StaticBounds, runtime_bounds<_BoundsTp> __rb) noexcept
: __arg_{::cuda::std::move(__arg)}
, __runtime_bounds_{__runtime_bound_cast<__element_type>(__rb.lower()),
__runtime_bound_cast<__element_type>(__rb.upper())}
{
__validate_bounds_intersection<__element_type, _StaticBounds>(__runtime_bounds_);
}
template <class _BoundsTp>
_CCCL_API constexpr __deferred_base(_Arg __arg, runtime_bounds<_BoundsTp> __rb, _StaticBounds __sb) noexcept
: __deferred_base(::cuda::std::move(__arg), __sb, __rb)
{}
};
//! @brief Wraps a reference to a single value that is potentially not available at API call time but will be available
//! by the time the argument is consumed in stream order.
template <class _Arg, class _StaticBounds = no_bounds>
class deferred : public __deferred_base<_Arg, _StaticBounds>
{
public:
using __deferred_base<_Arg, _StaticBounds>::__deferred_base;
};
#ifndef _CCCL_DOXYGEN_INVOKED
template <class _Arg>
_CCCL_HOST_DEVICE deferred(_Arg) -> deferred<_Arg>;
template <class _Arg, auto _Lowest, auto _Highest>
_CCCL_HOST_DEVICE deferred(_Arg, static_bounds<_Lowest, _Highest>) -> deferred<_Arg, static_bounds<_Lowest, _Highest>>;
template <class _Arg, class _Tp>
_CCCL_HOST_DEVICE deferred(_Arg, runtime_bounds<_Tp>) -> deferred<_Arg>;
template <class _Arg, auto _Lowest, auto _Highest, class _Tp>
_CCCL_HOST_DEVICE deferred(_Arg, static_bounds<_Lowest, _Highest>, runtime_bounds<_Tp>)
-> deferred<_Arg, static_bounds<_Lowest, _Highest>>;
template <class _Arg, class _Tp, auto _Lowest, auto _Highest>
_CCCL_HOST_DEVICE deferred(_Arg, runtime_bounds<_Tp>, static_bounds<_Lowest, _Highest>)
-> deferred<_Arg, static_bounds<_Lowest, _Highest>>;
#endif // _CCCL_DOXYGEN_INVOKED
//! @brief Wraps a reference to a sequence of values that is potentially not available at API call time but will be
//! available by the time the argument is consumed in stream order.
template <class _Arg, class _StaticBounds = no_bounds>
class deferred_sequence : public __deferred_base<_Arg, _StaticBounds>
{
public:
static_assert(__is_sequence_v<_Arg>, "deferred sequence arguments must have a distinct element type");
using __deferred_base<_Arg, _StaticBounds>::__deferred_base;
};
#ifndef _CCCL_DOXYGEN_INVOKED
template <class _Arg>
_CCCL_HOST_DEVICE deferred_sequence(_Arg) -> deferred_sequence<_Arg>;
template <class _Arg, auto _Lowest, auto _Highest>
_CCCL_HOST_DEVICE deferred_sequence(_Arg, static_bounds<_Lowest, _Highest>)
-> deferred_sequence<_Arg, static_bounds<_Lowest, _Highest>>;
template <class _Arg, class _Tp>
_CCCL_HOST_DEVICE deferred_sequence(_Arg, runtime_bounds<_Tp>) -> deferred_sequence<_Arg>;
template <class _Arg, auto _Lowest, auto _Highest, class _Tp>
_CCCL_HOST_DEVICE deferred_sequence(_Arg, static_bounds<_Lowest, _Highest>, runtime_bounds<_Tp>)
-> deferred_sequence<_Arg, static_bounds<_Lowest, _Highest>>;
template <class _Arg, class _Tp, auto _Lowest, auto _Highest>
_CCCL_HOST_DEVICE deferred_sequence(_Arg, runtime_bounds<_Tp>, static_bounds<_Lowest, _Highest>)
-> deferred_sequence<_Arg, static_bounds<_Lowest, _Highest>>;
#endif // _CCCL_DOXYGEN_INVOKED
// =====================================================================
// __access
// =====================================================================
struct __access
{
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API static constexpr _Arg& __arg(immediate<_Arg, _StaticBounds>& __wrapper) noexcept
{
return __wrapper.__arg_;
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API static constexpr const _Arg& __arg(const immediate<_Arg, _StaticBounds>& __wrapper) noexcept
{
return __wrapper.__arg_;
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API static constexpr _Arg&& __arg(immediate<_Arg, _StaticBounds>&& __wrapper) noexcept
{
return ::cuda::std::move(__wrapper.__arg_);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API static constexpr _Arg& __arg(__immediate_sequence<_Arg, _StaticBounds>& __wrapper) noexcept
{
return __wrapper.__arg_;
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API static constexpr const _Arg&
__arg(const __immediate_sequence<_Arg, _StaticBounds>& __wrapper) noexcept
{
return __wrapper.__arg_;
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API static constexpr _Arg&& __arg(__immediate_sequence<_Arg, _StaticBounds>&& __wrapper) noexcept
{
return ::cuda::std::move(__wrapper.__arg_);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API static constexpr _Arg& __arg(__deferred_base<_Arg, _StaticBounds>& __wrapper) noexcept
{
return __wrapper.__arg_;
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API static constexpr const _Arg&
__arg(const __deferred_base<_Arg, _StaticBounds>& __wrapper) noexcept
{
return __wrapper.__arg_;
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API static constexpr _Arg&& __arg(__deferred_base<_Arg, _StaticBounds>&& __wrapper) noexcept
{
return ::cuda::std::move(__wrapper.__arg_);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API static constexpr runtime_bounds<__element_type_of_t<_Arg>>&
__runtime_bounds(__immediate_sequence<_Arg, _StaticBounds>& __wrapper) noexcept
{
return __wrapper.__runtime_bounds_;
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API static constexpr const runtime_bounds<__element_type_of_t<_Arg>>&
__runtime_bounds(const __immediate_sequence<_Arg, _StaticBounds>& __wrapper) noexcept
{
return __wrapper.__runtime_bounds_;
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API static constexpr runtime_bounds<__element_type_of_t<_Arg>>&
__runtime_bounds(__deferred_base<_Arg, _StaticBounds>& __wrapper) noexcept
{
return __wrapper.__runtime_bounds_;
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API static constexpr const runtime_bounds<__element_type_of_t<_Arg>>&
__runtime_bounds(const __deferred_base<_Arg, _StaticBounds>& __wrapper) noexcept
{
return __wrapper.__runtime_bounds_;
}
};
// =====================================================================
// __unwrap
// =====================================================================
template <class _Tp>
inline constexpr bool __is_wrapper_v = false;
template <class _Arg, class _StaticBounds>
inline constexpr bool __is_wrapper_v<immediate<_Arg, _StaticBounds>> = true;
template <auto _Value, class _Tp>
inline constexpr bool __is_wrapper_v<constant<_Value, _Tp>> = true;
template <auto _Value>
inline constexpr bool __is_wrapper_v<__constant_sequence<_Value>> = true;
template <class _Arg, class _StaticBounds>
inline constexpr bool __is_wrapper_v<__immediate_sequence<_Arg, _StaticBounds>> = true;
template <class _Arg, class _StaticBounds>
inline constexpr bool __is_wrapper_v<deferred<_Arg, _StaticBounds>> = true;
template <class _Arg, class _StaticBounds>
inline constexpr bool __is_wrapper_v<deferred_sequence<_Arg, _StaticBounds>> = true;
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES((!__is_wrapper_v<::cuda::std::remove_cvref_t<_Tp>>) )
[[nodiscard]] _CCCL_API constexpr _Tp&& __unwrap(_Tp&& __arg) noexcept
{
return ::cuda::std::forward<_Tp>(__arg);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr _Arg& __unwrap(immediate<_Arg, _StaticBounds>& __arg) noexcept
{
return __access::__arg(__arg);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr const _Arg& __unwrap(const immediate<_Arg, _StaticBounds>& __arg) noexcept
{
return __access::__arg(__arg);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr _Arg __unwrap(immediate<_Arg, _StaticBounds>&& __arg) noexcept
{
return __access::__arg(::cuda::std::move(__arg));
}
template <auto _Value, class _Tp>
[[nodiscard]] _CCCL_API constexpr typename constant<_Value, _Tp>::value_type
__unwrap(const constant<_Value, _Tp>&) noexcept
{
return constant<_Value, _Tp>::__get_value();
}
template <auto _Value>
[[nodiscard]] _CCCL_API constexpr ::cuda::std::remove_cvref_t<decltype(_Value)>
__unwrap(const __constant_sequence<_Value>&) noexcept
{
return _Value;
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr _Arg& __unwrap(__immediate_sequence<_Arg, _StaticBounds>& __arg) noexcept
{
return __access::__arg(__arg);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr const _Arg& __unwrap(const __immediate_sequence<_Arg, _StaticBounds>& __arg) noexcept
{
return __access::__arg(__arg);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr _Arg __unwrap(__immediate_sequence<_Arg, _StaticBounds>&& __arg) noexcept
{
return __access::__arg(::cuda::std::move(__arg));
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr _Arg& __unwrap(deferred<_Arg, _StaticBounds>& __arg) noexcept
{
return __access::__arg(__arg);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr const _Arg& __unwrap(const deferred<_Arg, _StaticBounds>& __arg) noexcept
{
return __access::__arg(__arg);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr _Arg __unwrap(deferred<_Arg, _StaticBounds>&& __arg) noexcept
{
return __access::__arg(::cuda::std::move(__arg));
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr _Arg& __unwrap(deferred_sequence<_Arg, _StaticBounds>& __arg) noexcept
{
return __access::__arg(__arg);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr const _Arg& __unwrap(const deferred_sequence<_Arg, _StaticBounds>& __arg) noexcept
{
return __access::__arg(__arg);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr _Arg __unwrap(deferred_sequence<_Arg, _StaticBounds>&& __arg) noexcept
{
return __access::__arg(::cuda::std::move(__arg));
}
template <auto _Value, class _Tp>
_CCCL_API constexpr auto __constant_compute_lowest() noexcept
{
return constant<_Value, _Tp>::__get_value();
}
template <auto _Value, class _Tp>
_CCCL_API constexpr auto __constant_compute_highest() noexcept
{
return constant<_Value, _Tp>::__get_value();
}
template <auto _Value>
_CCCL_API constexpr auto __constant_sequence_compute_lowest() noexcept
{
using _ElementType = __element_type_of_t<::cuda::std::remove_cvref_t<decltype(_Value)>>;
auto __first = _Value.begin();
auto __last = _Value.end();
if (__first == __last)
{
return __type_lowest<_ElementType>();
}
return static_cast<_ElementType>(*::cuda::std::min_element(__first, __last));
}
template <auto _Value>
_CCCL_API constexpr auto __constant_sequence_compute_highest() noexcept
{
using _ElementType = __element_type_of_t<::cuda::std::remove_cvref_t<decltype(_Value)>>;
auto __first = _Value.begin();
auto __last = _Value.end();
if (__first == __last)
{
return __type_highest<_ElementType>();
}
return static_cast<_ElementType>(*::cuda::std::max_element(__first, __last));
}
// =====================================================================
// __traits
// =====================================================================
//! @brief Traits for argument wrappers and plain argument values.
//!
//! Models @c numeric_limits for bounds: @c lowest is the lower bound, @c highest is the upper bound.
//! Use in @c if @c constexpr for compile-time dispatch based on bounds.
template <class _Tp>
struct __traits_impl
{
using value_type = _Tp;
using element_type = __element_type_of_t<_Tp>;
static constexpr bool is_constant = false;
static constexpr bool is_deferred = false;
static constexpr bool is_single_value = true;
static constexpr element_type lowest = __type_lowest<element_type>();
static constexpr element_type highest = __type_highest<element_type>();
};
template <auto _Value, class _Tp>
struct __traits_impl<constant<_Value, _Tp>>
{
using value_type = typename constant<_Value, _Tp>::value_type;
using element_type = value_type;
static constexpr bool is_constant = true;
static constexpr bool is_deferred = false;
static constexpr bool is_single_value = true;
static constexpr element_type lowest = __constant_compute_lowest<_Value, _Tp>();
static constexpr element_type highest = __constant_compute_highest<_Value, _Tp>();
};
template <class _Arg, class _StaticBounds>
struct __traits_impl<immediate<_Arg, _StaticBounds>>
{
using value_type = _Arg;
using element_type = __element_type_of_t<_Arg>;
static_assert(__valid_static_bounds_v<element_type, _StaticBounds>,
"argument wrapper bounds type must be cuda::args::no_bounds or cuda::args::static_bounds with "
"values representable by the element type");
static constexpr bool is_constant = false;
static constexpr bool is_deferred = false;
static constexpr bool is_single_value = true;
static constexpr element_type lowest = __wrapper_static_lowest<element_type, _StaticBounds>();
static constexpr element_type highest = __wrapper_static_highest<element_type, _StaticBounds>();
};
template <auto _Value>
struct __traits_impl<__constant_sequence<_Value>>
{
using value_type = ::cuda::std::remove_cvref_t<decltype(_Value)>;
using element_type = __element_type_of_t<value_type>;
static_assert(__is_sequence_v<value_type>, "The value type of __constant_sequence must be a sequence");
static constexpr bool is_constant = true;
static constexpr bool is_deferred = false;
static constexpr bool is_single_value = false;
static constexpr element_type lowest = __constant_sequence_compute_lowest<_Value>();
static constexpr element_type highest = __constant_sequence_compute_highest<_Value>();
};
template <class _Arg, class _StaticBounds>
struct __traits_impl<__immediate_sequence<_Arg, _StaticBounds>>
{
using value_type = _Arg;
using element_type = __element_type_of_t<_Arg>;
static_assert(__is_sequence_v<value_type>, "immediate sequence arguments must have a distinct element type");
static_assert(__valid_static_bounds_v<element_type, _StaticBounds>,
"argument wrapper bounds type must be cuda::args::no_bounds or cuda::args::static_bounds with "
"values representable by the element type");
static constexpr bool is_constant = false;
static constexpr bool is_deferred = false;
static constexpr bool is_single_value = false;
static constexpr element_type lowest = __wrapper_static_lowest<element_type, _StaticBounds>();
static constexpr element_type highest = __wrapper_static_highest<element_type, _StaticBounds>();
};
template <class _Arg, class _StaticBounds>
struct __traits_impl<deferred<_Arg, _StaticBounds>>
{
using value_type = _Arg;
using element_type = __element_type_of_t<_Arg>;
static_assert(__valid_static_bounds_v<element_type, _StaticBounds>,
"argument wrapper bounds type must be cuda::args::no_bounds or cuda::args::static_bounds with "
"values representable by the element type");
static constexpr bool is_constant = false;
static constexpr bool is_deferred = true;
static constexpr bool is_single_value = true;
static constexpr element_type lowest = __wrapper_static_lowest<element_type, _StaticBounds>();
static constexpr element_type highest = __wrapper_static_highest<element_type, _StaticBounds>();
};
template <class _Arg, class _StaticBounds>
struct __traits_impl<deferred_sequence<_Arg, _StaticBounds>>
{
using value_type = _Arg;
using element_type = __element_type_of_t<_Arg>;
static_assert(__is_sequence_v<value_type>, "deferred sequence arguments must have a distinct element type");
static_assert(__valid_static_bounds_v<element_type, _StaticBounds>,
"argument wrapper bounds type must be cuda::args::no_bounds or cuda::args::static_bounds with "
"values representable by the element type");
static constexpr bool is_constant = false;
static constexpr bool is_deferred = true;
static constexpr bool is_single_value = false;
static constexpr element_type lowest = __wrapper_static_lowest<element_type, _StaticBounds>();
static constexpr element_type highest = __wrapper_static_highest<element_type, _StaticBounds>();
};
template <class _Tp>
struct __traits : __traits_impl<::cuda::std::remove_cvref_t<_Tp>>
{};
// =====================================================================
// __lowest_ / __highest_ — free functions
// =====================================================================
//! @brief Returns the effective lowest bound, combining static and runtime bounds.
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES((!__is_wrapper_v<::cuda::std::remove_cv_t<_Tp>>) )
[[nodiscard]] _CCCL_API constexpr auto __lowest_(_Tp) noexcept
{
return __type_lowest<__element_type_of_t<_Tp>>();
}
template <auto _Value, class _Tp>
[[nodiscard]] _CCCL_API constexpr auto __lowest_(constant<_Value, _Tp>) noexcept
{
return __constant_compute_lowest<_Value, _Tp>();
}
template <auto _Value>
[[nodiscard]] _CCCL_API constexpr auto __lowest_(__constant_sequence<_Value>) noexcept
{
return __constant_sequence_compute_lowest<_Value>();
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr auto __lowest_(immediate<_Arg, _StaticBounds> __arg) noexcept
{
return __access::__arg(__arg);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr auto __lowest_(__immediate_sequence<_Arg, _StaticBounds> __arg) noexcept
{
using _ET = __element_type_of_t<_Arg>;
const auto& __runtime_bounds = __access::__runtime_bounds(__arg);
__validate_bounds_intersection<_ET, _StaticBounds>(__runtime_bounds);
return __effective_lowest<_ET, _StaticBounds>(__runtime_bounds);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr auto __lowest_(deferred<_Arg, _StaticBounds> __arg) noexcept
{
using _ET = __element_type_of_t<_Arg>;
const auto& __runtime_bounds = __access::__runtime_bounds(__arg);
__validate_bounds_intersection<_ET, _StaticBounds>(__runtime_bounds);
return __effective_lowest<_ET, _StaticBounds>(__runtime_bounds);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr auto __lowest_(deferred_sequence<_Arg, _StaticBounds> __arg) noexcept
{
using _ET = __element_type_of_t<_Arg>;
const auto& __runtime_bounds = __access::__runtime_bounds(__arg);
__validate_bounds_intersection<_ET, _StaticBounds>(__runtime_bounds);
return __effective_lowest<_ET, _StaticBounds>(__runtime_bounds);
}
//! @brief Returns the effective highest bound, combining static and runtime bounds.
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES((!__is_wrapper_v<::cuda::std::remove_cv_t<_Tp>>) )
[[nodiscard]] _CCCL_API constexpr auto __highest_(_Tp) noexcept
{
return __type_highest<__element_type_of_t<_Tp>>();
}
template <auto _Value, class _Tp>
[[nodiscard]] _CCCL_API constexpr auto __highest_(constant<_Value, _Tp>) noexcept
{
return __constant_compute_highest<_Value, _Tp>();
}
template <auto _Value>
[[nodiscard]] _CCCL_API constexpr auto __highest_(__constant_sequence<_Value>) noexcept
{
return __constant_sequence_compute_highest<_Value>();
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr auto __highest_(immediate<_Arg, _StaticBounds> __arg) noexcept
{
return __access::__arg(__arg);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr auto __highest_(__immediate_sequence<_Arg, _StaticBounds> __arg) noexcept
{
using _ET = __element_type_of_t<_Arg>;
const auto& __runtime_bounds = __access::__runtime_bounds(__arg);
__validate_bounds_intersection<_ET, _StaticBounds>(__runtime_bounds);
return __effective_highest<_ET, _StaticBounds>(__runtime_bounds);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr auto __highest_(deferred<_Arg, _StaticBounds> __arg) noexcept
{
using _ET = __element_type_of_t<_Arg>;
const auto& __runtime_bounds = __access::__runtime_bounds(__arg);
__validate_bounds_intersection<_ET, _StaticBounds>(__runtime_bounds);
return __effective_highest<_ET, _StaticBounds>(__runtime_bounds);
}
template <class _Arg, class _StaticBounds>
[[nodiscard]] _CCCL_API constexpr auto __highest_(deferred_sequence<_Arg, _StaticBounds> __arg) noexcept
{
using _ET = __element_type_of_t<_Arg>;
const auto& __runtime_bounds = __access::__runtime_bounds(__arg);
__validate_bounds_intersection<_ET, _StaticBounds>(__runtime_bounds);
return __effective_highest<_ET, _StaticBounds>(__runtime_bounds);
}
_CCCL_END_NAMESPACE_CUDA_ARGUMENT
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ARGUMENT_ARGUMENT_H

195
qwen3_6_scripts/cccl_preload/include/cuda/__argument/argument_bounds.h
View File

@@ -1,195 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ARGUMENT_ARGUMENT_BOUNDS_H
#define _CUDA___ARGUMENT_ARGUMENT_BOUNDS_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__cccl/assert.h>
#include <cuda/std/__type_traits/is_same.h>
#include <cuda/std/__type_traits/remove_cvref.h>
#include <cuda/std/limits>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_ARGUMENT
template <class _Lhs, class _Rhs>
[[nodiscard]] _CCCL_API constexpr bool __bounds_less_equal(const _Lhs& __lhs, const _Rhs& __rhs) noexcept
{
return (__lhs < __rhs) || (__lhs == __rhs);
}
template <class _Lhs, class _Rhs>
[[nodiscard]] _CCCL_API constexpr bool __bounds_greater_equal(const _Lhs& __lhs, const _Rhs& __rhs) noexcept
{
return (__rhs < __lhs) || (__lhs == __rhs);
}
//! @brief Sentinel type indicating no bounds are present.
struct no_bounds
{};
// =====================================================================
// static_bounds
// =====================================================================
//! @brief Compile-time bounds on an argument value.
//!
//! The bound type is deduced from the non-type template parameters.
//!
//! @tparam _Lower The static lower bound.
//! @tparam _Upper The static upper bound.
template <auto _Lower, auto _Upper>
class static_bounds
{
public:
static_assert(::cuda::std::is_same_v<decltype(_Lower), decltype(_Upper)>,
"Static bounds endpoints must have the same type");
static_assert(__bounds_less_equal(_Lower, _Upper), "Lower bound must be <= upper bound");
[[nodiscard]] _CCCL_API static constexpr decltype(_Lower) lower() noexcept
{
return _Lower;
}
[[nodiscard]] _CCCL_API static constexpr decltype(_Upper) upper() noexcept
{
return _Upper;
}
};
template <class _Tp>
inline constexpr bool __is_static_bounds_v = false;
template <auto _Lower, auto _Upper>
inline constexpr bool __is_static_bounds_v<static_bounds<_Lower, _Upper>> = true;
// =====================================================================
// __type_lowest / __type_highest
// =====================================================================
// The implicit bounds of an element type are derived from cuda::std::numeric_limits. The primary numeric_limits
// template returns a value-initialized object from lowest()/max() and is therefore meaningless as a bound, so require
// an explicit specialization rather than silently producing a degenerate range.
//! @brief Returns the lowest value representable by the element type @c _Tp.
//! @tparam _Tp The element type. It must have a @c cuda::std::numeric_limits specialization.
//! @return @c cuda::std::numeric_limits<_Tp>::lowest().
template <class _Tp>
[[nodiscard]] _CCCL_API constexpr _Tp __type_lowest() noexcept
{
static_assert(::cuda::std::numeric_limits<_Tp>::is_specialized,
"cuda::args bounds require a specialized cuda::std::numeric_limits for the element type. Provide "
"explicit bounds for element types without a numeric_limits specialization.");
return ::cuda::std::numeric_limits<_Tp>::lowest();
}
//! @brief Returns the highest value representable by the element type @c _Tp.
//! @tparam _Tp The element type. It must have a @c cuda::std::numeric_limits specialization.
//! @return @c cuda::std::numeric_limits<_Tp>::max().
template <class _Tp>
[[nodiscard]] _CCCL_API constexpr _Tp __type_highest() noexcept
{
static_assert(::cuda::std::numeric_limits<_Tp>::is_specialized,
"cuda::args bounds require a specialized cuda::std::numeric_limits for the element type. Provide "
"explicit bounds for element types without a numeric_limits specialization.");
return (::cuda::std::numeric_limits<_Tp>::max)();
}
// =====================================================================
// runtime_bounds
// =====================================================================
//! @brief Runtime bounds on an argument value.
//!
//! @tparam _Tp The value type of the bounds.
template <class _Tp>
class runtime_bounds
{
_Tp __lower_ = __type_lowest<_Tp>();
_Tp __upper_ = __type_highest<_Tp>();
public:
constexpr runtime_bounds() noexcept = default;
_CCCL_API constexpr runtime_bounds(_Tp __lower, _Tp __upper) noexcept
: __lower_(__lower)
, __upper_(__upper)
{
_CCCL_ASSERT(__bounds_less_equal(__lower, __upper), "Runtime lower bound must be <= runtime upper bound");
}
[[nodiscard]] _CCCL_API constexpr _Tp lower() const noexcept
{
return __lower_;
}
[[nodiscard]] _CCCL_API constexpr _Tp upper() const noexcept
{
return __upper_;
}
};
#ifndef _CCCL_DOXYGEN_INVOKED
template <class _Tp>
_CCCL_HOST_DEVICE runtime_bounds(_Tp, _Tp) -> runtime_bounds<_Tp>;
#endif // _CCCL_DOXYGEN_INVOKED
template <class _Tp>
inline constexpr bool __is_runtime_bounds_v = false;
template <class _Tp>
inline constexpr bool __is_runtime_bounds_v<runtime_bounds<_Tp>> = true;
// =====================================================================
// bounds — factory functions
// =====================================================================
//! @brief Create compile-time bounds.
//!
//! @tparam _Lower The static lower bound.
//! @tparam _Upper The static upper bound.
//! @return A compile-time bounds object.
template <auto _Lower, auto _Upper>
[[nodiscard]] _CCCL_API constexpr static_bounds<_Lower, _Upper> bounds() noexcept
{
return {};
}
//! @brief Create runtime bounds.
//!
//! @param __lower The runtime lower bound.
//! @param __upper The runtime upper bound.
//! @return A runtime bounds object.
template <class _Tp>
[[nodiscard]] _CCCL_API constexpr runtime_bounds<_Tp> bounds(_Tp __lower, _Tp __upper) noexcept
{
return {__lower, __upper};
}
template <class _Tp>
inline constexpr bool __is_static_bounds_cv_v = __is_static_bounds_v<::cuda::std::remove_cvref_t<_Tp>>;
template <class _Tp>
inline constexpr bool __is_runtime_bounds_cv_v = __is_runtime_bounds_v<::cuda::std::remove_cvref_t<_Tp>>;
template <class _Tp>
inline constexpr bool __is_bounds_v = __is_static_bounds_cv_v<_Tp> || __is_runtime_bounds_cv_v<_Tp>;
_CCCL_END_NAMESPACE_CUDA_ARGUMENT
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ARGUMENT_ARGUMENT_BOUNDS_H

151
qwen3_6_scripts/cccl_preload/include/cuda/__atomic/atomic.h
View File

@@ -1,151 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ATOMIC_ATOMIC_H
#define _CUDA___ATOMIC_ATOMIC_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__type_traits/copy_cv.h>
#include <cuda/std/atomic>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
// atomic<T>
template <class _Tp, thread_scope _Sco = thread_scope::thread_scope_system>
struct atomic : public ::cuda::std::__atomic_impl<_Tp, _Sco>
{
using value_type = _Tp;
_CCCL_HIDE_FROM_ABI constexpr atomic() noexcept = default;
_CCCL_HOST_DEVICE_API constexpr atomic(_Tp __d) noexcept
: ::cuda::std::__atomic_impl<_Tp, _Sco>(__d)
{}
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
_CCCL_HOST_DEVICE_API inline _Tp operator=(_Tp __d) volatile noexcept
{
this->store(__d);
return __d;
}
_CCCL_HOST_DEVICE_API inline _Tp operator=(_Tp __d) noexcept
{
this->store(__d);
return __d;
}
_CCCL_HOST_DEVICE_API inline _Tp fetch_max(const _Tp& __op, memory_order __m = memory_order_seq_cst) noexcept
{
return ::cuda::std::__atomic_fetch_max_dispatch(&this->__a, __op, __m, ::cuda::std::__scope_to_tag<_Sco>{});
}
_CCCL_HOST_DEVICE_API inline _Tp fetch_max(const _Tp& __op, memory_order __m = memory_order_seq_cst) volatile noexcept
{
return ::cuda::std::__atomic_fetch_max_dispatch(&this->__a, __op, __m, ::cuda::std::__scope_to_tag<_Sco>{});
}
_CCCL_HOST_DEVICE_API inline _Tp fetch_min(const _Tp& __op, memory_order __m = memory_order_seq_cst) noexcept
{
return ::cuda::std::__atomic_fetch_min_dispatch(&this->__a, __op, __m, ::cuda::std::__scope_to_tag<_Sco>{});
}
_CCCL_HOST_DEVICE_API inline _Tp fetch_min(const _Tp& __op, memory_order __m = memory_order_seq_cst) volatile noexcept
{
return ::cuda::std::__atomic_fetch_min_dispatch(&this->__a, __op, __m, ::cuda::std::__scope_to_tag<_Sco>{});
}
};
// atomic_ref<T>
template <class _Tp, thread_scope _Sco = thread_scope::thread_scope_system>
struct atomic_ref : public ::cuda::std::__atomic_ref_impl<_Tp, _Sco>
{
using value_type = _Tp;
static constexpr size_t required_alignment = sizeof(_Tp);
static constexpr bool is_always_lock_free = sizeof(_Tp) <= 8;
_CCCL_HOST_DEVICE_API explicit constexpr atomic_ref(_Tp& __ref)
: ::cuda::std::__atomic_ref_impl<_Tp, _Sco>(__ref)
{}
_CCCL_HOST_DEVICE_API inline _Tp operator=(_Tp __v) const noexcept
{
this->store(__v);
return __v;
}
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::__copy_cv_t<_Tp, void>* address() const noexcept
{
return this->__a.get();
}
_CCCL_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
atomic_ref& operator=(const atomic_ref&) = delete;
atomic_ref& operator=(const atomic_ref&) const = delete;
_CCCL_HOST_DEVICE_API inline _Tp fetch_max(const _Tp& __op, memory_order __m = memory_order_seq_cst) const noexcept
{
return ::cuda::std::__atomic_fetch_max_dispatch(&this->__a, __op, __m, ::cuda::std::__scope_to_tag<_Sco>{});
}
_CCCL_HOST_DEVICE_API inline _Tp fetch_min(const _Tp& __op, memory_order __m = memory_order_seq_cst) const noexcept
{
return ::cuda::std::__atomic_fetch_min_dispatch(&this->__a, __op, __m, ::cuda::std::__scope_to_tag<_Sco>{});
}
};
_CCCL_HOST_DEVICE_API inline void
atomic_thread_fence(memory_order __m, [[maybe_unused]] thread_scope _Scope = thread_scope::thread_scope_system)
{
NV_DISPATCH_TARGET(
NV_IS_DEVICE,
(switch (_Scope) {
case thread_scope::thread_scope_system:
::cuda::std::__atomic_thread_fence_cuda((int) __m, __thread_scope_system_tag{});
break;
case thread_scope::thread_scope_device:
::cuda::std::__atomic_thread_fence_cuda((int) __m, __thread_scope_device_tag{});
break;
case thread_scope::thread_scope_block:
::cuda::std::__atomic_thread_fence_cuda((int) __m, __thread_scope_block_tag{});
break;
// Atomics scoped to themselves do not require fencing
case thread_scope::thread_scope_thread:
break;
}),
NV_IS_HOST,
(::cuda::std::atomic_thread_fence(__m);))
}
_CCCL_HOST_DEVICE_API inline void atomic_signal_fence(memory_order __m)
{
::cuda::std::atomic_signal_fence(__m);
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ATOMIC_ATOMIC_H

129
qwen3_6_scripts/cccl_preload/include/cuda/__bit/bitfield.h
View File

@@ -1,129 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___BIT_BITFILED_INSERT_EXTRACT_H
#define _CUDA___BIT_BITFILED_INSERT_EXTRACT_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__bit/bitmask.h>
#include <cuda/std/__bit/shl.h>
#include <cuda/std/__bit/shr.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/conditional.h>
#include <cuda/std/__type_traits/is_unsigned_integer.h>
#include <cuda/std/cstdint>
#include <cuda/std/limits>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
#if !_CCCL_TILE_COMPILATION() // error: asm statement is unsupported in tile code
# if __cccl_ptx_isa >= 200
[[nodiscard]]
_CCCL_DEVICE_API inline uint32_t __bfi(uint32_t __dest, uint32_t __source, int __start, int __width) noexcept
{
asm("bfi.b32 %0, %1, %2, %3, %4;" : "=r"(__dest) : "r"(__source), "r"(__dest), "r"(__start), "r"(__width));
return __dest;
}
[[nodiscard]] _CCCL_DEVICE_API inline uint64_t
__bfi(uint64_t __dest, uint64_t __source, int __start, int __width) noexcept
{
asm("bfi.b64 %0, %1, %2, %3, %4;" : "=l"(__dest) : "l"(__source), "l"(__dest), "r"(__start), "r"(__width));
return __dest;
}
[[nodiscard]] _CCCL_DEVICE_API inline uint32_t __bfe(uint32_t __value, int __start, int __width) noexcept
{
uint32_t __ret;
asm("bfe.u32 %0, %1, %2, %3;" : "=r"(__ret) : "r"(__value), "r"(__start), "r"(__width));
return __ret;
}
[[nodiscard]] _CCCL_DEVICE_API inline uint64_t __bfe(uint64_t __value, int __start, int __width) noexcept
{
uint64_t __ret;
asm("bfe.u64 %0, %1, %2, %3;" : "=l"(__ret) : "l"(__value), "r"(__start), "r"(__width));
return __ret;
}
# endif // __cccl_ptx_isa >= 200
#endif // !_CCCL_TILE_COMPILATION()
template <typename _Tp>
[[nodiscard]]
_CCCL_API constexpr _Tp bitfield_insert(const _Tp __dest, const _Tp __source, int __start, int __width) noexcept
{
static_assert(::cuda::std::__cccl_is_cv_unsigned_integer_v<_Tp>, "bitfield_insert() requires unsigned integer types");
[[maybe_unused]] constexpr auto __digits = ::cuda::std::numeric_limits<_Tp>::digits;
_CCCL_ASSERT(__width >= 0 && __width <= __digits, "width out of range");
_CCCL_ASSERT(__start >= 0 && __start <= __digits, "start position out of range");
_CCCL_ASSERT(__start + __width <= __digits, "start position + width out of range");
#if !_CCCL_TILE_COMPILATION() // error: asm statement is unsupported in tile code
_CCCL_IF_NOT_CONSTEVAL_DEFAULT
{
if constexpr (sizeof(_Tp) <= sizeof(uint64_t))
{
// clang-format off
NV_DISPATCH_TARGET( // all SM < 70
NV_PROVIDES_SM_70, (;),
NV_IS_DEVICE, (using _Up = ::cuda::std::_If<sizeof(_Tp) <= sizeof(uint32_t), uint32_t, uint64_t>;
return ::cuda::__bfi(static_cast<_Up>(__dest), static_cast<_Up>(__source),
__start, __width);))
// clang-format on
}
}
#endif // !_CCCL_TILE_COMPILATION()
auto __mask = ::cuda::bitmask<_Tp>(__start, __width);
return (::cuda::std::shl(__source, static_cast<unsigned>(__start)) & __mask) | (__dest & ~__mask);
}
template <typename _Tp>
[[nodiscard]] _CCCL_API constexpr _Tp bitfield_extract(const _Tp __value, int __start, int __width) noexcept
{
static_assert(::cuda::std::__cccl_is_cv_unsigned_integer_v<_Tp>,
"bitfield_extract() requires unsigned integer types");
[[maybe_unused]] constexpr auto __digits = ::cuda::std::numeric_limits<_Tp>::digits;
_CCCL_ASSERT(__width >= 0 && __width <= __digits, "width out of range");
_CCCL_ASSERT(__start >= 0 && __start <= __digits, "start position out of range");
_CCCL_ASSERT(__start + __width <= __digits, "start position + width out of range");
#if !_CCCL_TILE_COMPILATION() // error: asm statement is unsupported in tile code
_CCCL_IF_NOT_CONSTEVAL_DEFAULT
{
if constexpr (sizeof(_Tp) <= sizeof(uint32_t))
{
// clang-format off
NV_DISPATCH_TARGET( // all SM < 70
NV_PROVIDES_SM_70, (;),
NV_IS_DEVICE, (using _Up = ::cuda::std::_If<sizeof(_Tp) <= sizeof(uint32_t), uint32_t, uint64_t>;
return ::cuda::__bfe(static_cast<_Up>(__value), __start, __width);))
// clang-format on
}
}
#endif // !_CCCL_TILE_COMPILATION()
return ::cuda::std::shr(__value, static_cast<unsigned>(__start)) & ::cuda::bitmask<_Tp>(0, __width);
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___BIT_BITFILED_INSERT_EXTRACT_H

62
qwen3_6_scripts/cccl_preload/include/cuda/__bit/bitmask.h
View File

@@ -1,62 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___BIT_BITMASK_H
#define _CUDA___BIT_BITMASK_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__bit/shl.h>
#include <cuda/std/__type_traits/conditional.h>
#include <cuda/std/__type_traits/is_unsigned_integer.h>
#include <cuda/std/limits>
#if _CCCL_CUDA_COMPILATION() && !_CCCL_TILE_COMPILATION()
# include <cuda/__ptx/instructions/bmsk.h>
#endif // _CCCL_CUDA_COMPILATION() && !_CCCL_TILE_COMPILATION()
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <typename _Tp = uint32_t>
[[nodiscard]] _CCCL_API constexpr _Tp bitmask(int __start, int __width) noexcept
{
static_assert(::cuda::std::__cccl_is_unsigned_integer_v<_Tp>, "bitmask() requires unsigned integer types");
[[maybe_unused]] constexpr auto __digits = ::cuda::std::numeric_limits<_Tp>::digits;
_CCCL_ASSERT(__width >= 0 && __width <= __digits, "width out of range");
_CCCL_ASSERT(__start >= 0 && __start <= __digits, "start position out of range");
_CCCL_ASSERT(__start + __width <= __digits, "start position + width out of range");
#if !_CCCL_TILE_COMPILATION() // error: asm statement is unsupported in tile code
_CCCL_IF_NOT_CONSTEVAL_DEFAULT
{
if constexpr (sizeof(_Tp) <= sizeof(uint32_t))
{
NV_IF_TARGET(NV_PROVIDES_SM_70, (return ::cuda::ptx::bmsk_clamp(__start, __width);))
}
}
#endif // !_CCCL_TILE_COMPILATION()
return ::cuda::std::shl(static_cast<_Tp>(::cuda::std::shl(_Tp{1}, static_cast<unsigned>(__width)) - 1),
static_cast<unsigned>(__start));
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___BIT_BITMASK_H

203
qwen3_6_scripts/cccl_preload/include/cuda/__cmath/ilog.h
View File

@@ -1,203 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___CMATH_ILOG_H
#define _CUDA___CMATH_ILOG_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__bit/has_single_bit.h>
#include <cuda/std/__bit/integral.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_same.h>
#include <cuda/std/__type_traits/make_unsigned.h>
#include <cuda/std/array>
#include <cuda/std/cstdint>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
_CCCL_TEMPLATE(typename _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_cv_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr int ilog2(const _Tp __t) noexcept
{
using _Up = ::cuda::std::make_unsigned_t<_Tp>;
_CCCL_ASSERT(__t > 0, "ilog2() argument must be strictly positive");
auto __log2_approx = ::cuda::std::__bit_log2(static_cast<_Up>(__t));
_CCCL_ASSUME(__log2_approx <= ::cuda::std::numeric_limits<_Tp>::digits);
return __log2_approx;
}
_CCCL_TEMPLATE(typename _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_cv_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr int ceil_ilog2(const _Tp __t) noexcept
{
using _Up = ::cuda::std::make_unsigned_t<_Tp>;
return ::cuda::ilog2(__t) + !::cuda::std::has_single_bit(static_cast<_Up>(__t));
}
[[nodiscard]] _CCCL_API _CCCL_CONSTEVAL ::cuda::std::array<::cuda::std::uint32_t, 10> __power_of_10_32bit() noexcept
{
return {10,
100,
1'000,
10'000,
100'000,
1'000'000,
10'000'000,
100'000'000,
1'000'000'000,
::cuda::std::numeric_limits<::cuda::std::uint32_t>::max()};
}
[[nodiscard]] _CCCL_API _CCCL_CONSTEVAL ::cuda::std::array<::cuda::std::uint64_t, 20> __power_of_10_64bit() noexcept
{
return {
10,
100,
1'000,
10'000,
100'000,
1'000'000,
10'000'000,
100'000'000,
1'000'000'000,
10'000'000'000,
100'000'000'000,
1'000'000'000'000,
10'000'000'000'000,
100'000'000'000'000,
1'000'000'000'000'000,
10'000'000'000'000'000,
100'000'000'000'000'000,
1'000'000'000'000'000'000,
10'000'000'000'000'000'000ull,
::cuda::std::numeric_limits<::cuda::std::uint64_t>::max()};
}
#if _CCCL_HAS_INT128()
[[nodiscard]] _CCCL_API _CCCL_CONSTEVAL ::cuda::std::array<__uint128_t, 39> __power_of_10_128bit() noexcept
{
return {
10,
100,
1'000,
10'000,
100'000,
1'000'000,
10'000'000,
100'000'000,
1'000'000'000,
10'000'000'000,
100'000'000'000,
1'000'000'000'000,
10'000'000'000'000,
100'000'000'000'000,
1'000'000'000'000'000,
10'000'000'000'000'000,
100'000'000'000'000'000,
1'000'000'000'000'000'000,
10'000'000'000'000'000'000ull,
__uint128_t{10'000'000'000'000'000'000ull} * 10,
__uint128_t{10'000'000'000'000'000'000ull} * 100,
__uint128_t{10'000'000'000'000'000'000ull} * 1'000,
__uint128_t{10'000'000'000'000'000'000ull} * 10'000,
__uint128_t{10'000'000'000'000'000'000ull} * 100'000,
__uint128_t{10'000'000'000'000'000'000ull} * 1'000'000,
__uint128_t{10'000'000'000'000'000'000ull} * 10'000'000,
__uint128_t{10'000'000'000'000'000'000ull} * 100'000'000,
__uint128_t{10'000'000'000'000'000'000ull} * 1'000'000'000,
__uint128_t{10'000'000'000'000'000'000ull} * 10'000'000'000,
__uint128_t{10'000'000'000'000'000'000ull} * 100'000'000'000,
__uint128_t{10'000'000'000'000'000'000ull} * 1'000'000'000'000,
__uint128_t{10'000'000'000'000'000'000ull} * 10'000'000'000'000,
__uint128_t{10'000'000'000'000'000'000ull} * 100'000'000'000'000,
__uint128_t{10'000'000'000'000'000'000ull} * 1'000'000'000'000'000,
__uint128_t{10'000'000'000'000'000'000ull} * 1'000'000'000'000'0000,
__uint128_t{10'000'000'000'000'000'000ull} * 10'000'000'000'000'0000,
__uint128_t{10'000'000'000'000'000'000ull} * 100'000'000'000'000'0000,
__uint128_t{10'000'000'000'000'000'000ull} * 1'000'000'000'000'000'0000ull,
::cuda::std::numeric_limits<__uint128_t>::max()};
}
#endif // _CCCL_HAS_INT128()
_CCCL_TEMPLATE(typename _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_cv_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr int ilog10(const _Tp __t) noexcept
{
using ::cuda::std::uint32_t;
using ::cuda::std::uint64_t;
_CCCL_ASSERT(__t > 0, "cuda::ilog10() argument must be strictly positive");
constexpr auto __reciprocal_log2_10 = 0.301029995663f; // 1 / log2(10)
const auto __log2 = ::cuda::ilog2(__t) * __reciprocal_log2_10;
auto __log10_approx = static_cast<int>(__log2);
if constexpr (sizeof(_Tp) <= sizeof(uint32_t))
{
_CCCL_ASSERT(__log10_approx < static_cast<int>(::cuda::__power_of_10_32bit().size()), "out of bounds");
if constexpr (::cuda::std::is_same_v<_Tp, uint32_t>)
{
// don't replace +1 with >= because wraparound behavior is needed here
__log10_approx += static_cast<uint32_t>(__t) + 1 > ::cuda::__power_of_10_32bit()[__log10_approx];
}
else
{
__log10_approx += static_cast<uint32_t>(__t) >= ::cuda::__power_of_10_32bit()[__log10_approx];
}
}
else if constexpr (sizeof(_Tp) == sizeof(uint64_t))
{
_CCCL_ASSERT(__log10_approx < static_cast<int>(::cuda::__power_of_10_64bit().size()), "out of bounds");
// +1 is not needed here
__log10_approx += static_cast<uint64_t>(__t) >= ::cuda::__power_of_10_64bit()[__log10_approx];
}
#if _CCCL_HAS_INT128()
else
{
_CCCL_ASSERT(__log10_approx < static_cast<int>(::cuda::__power_of_10_128bit().size()), "out of bounds");
if constexpr (::cuda::std::is_same_v<_Tp, __uint128_t>)
{
// don't replace +1 with >= because wraparound behavior is needed here
__log10_approx += static_cast<__uint128_t>(__t) + 1 > ::cuda::__power_of_10_128bit()[__log10_approx];
}
else
{
__log10_approx += static_cast<__uint128_t>(__t) >= ::cuda::__power_of_10_128bit()[__log10_approx];
}
}
#endif // _CCCL_HAS_INT128()
_CCCL_ASSUME(__log10_approx <= ::cuda::std::numeric_limits<_Tp>::digits / 3); // 2^X < 10^(x/3) -> 8^X < 10^x
return __log10_approx;
}
_CCCL_TEMPLATE(typename _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_cv_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr int ceil_ilog10(const _Tp __t) noexcept
{
_CCCL_ASSERT(__t > 0, "cuda::ceil_ilog10() argument must be strictly positive");
return __t == 1 ? 0 : ::cuda::ilog10(static_cast<_Tp>(__t - 1)) + 1;
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___CMATH_ILOG_H

147
qwen3_6_scripts/cccl_preload/include/cuda/__cmath/mul_hi.h
View File

@@ -1,147 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___CMATH_MUL_HI_H
#define _CUDA___CMATH_MUL_HI_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__type_traits/make_nbit_int.h>
#include <cuda/std/__type_traits/make_unsigned.h>
#include <cuda/std/__type_traits/num_bits.h>
#include <cuda/std/cstdint>
#if _CCCL_COMPILER(MSVC)
# include <intrin.h>
#endif // _CCCL_COMPILER(MSVC)
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
/***********************************************************************************************************************
* Extract higher bits after multiplication
**********************************************************************************************************************/
template <typename _Tp>
[[nodiscard]] _CCCL_API constexpr _Tp __mul_hi_fallback(_Tp __lhs, _Tp __rhs) noexcept
{
static_assert(::cuda::std::is_unsigned_v<_Tp>, "__mul_hi_fallback: T is required to be a unsigned integer type");
constexpr int __half_bits = ::cuda::std::__num_bits_v<_Tp> / 2;
using __half_bits_t = ::cuda::std::__make_nbit_uint_t<__half_bits>;
const auto __lhs_low = static_cast<__half_bits_t>(__lhs); // 32-bit
const auto __lhs_high = static_cast<__half_bits_t>(__lhs >> __half_bits); // 32-bit
const auto __rhs_low = static_cast<__half_bits_t>(__rhs); // 32-bit
const auto __rhs_high = static_cast<__half_bits_t>(__rhs >> __half_bits); // 32-bit
const auto __po_half = (static_cast<_Tp>(__lhs_low) * __rhs_low) >> __half_bits;
const auto __p1 = static_cast<_Tp>(__lhs_low) * __rhs_high; // 64-bit
const auto __p2 = static_cast<_Tp>(__lhs_high) * __rhs_low; // 64-bit
const auto __p3 = static_cast<_Tp>(__lhs_high) * __rhs_high; // 64-bit
const auto __p1_half = static_cast<__half_bits_t>(__p1); // 32-bit
const auto __p2_half = static_cast<__half_bits_t>(__p2); // 32-bit
const auto __carry = (__po_half + __p1_half + __p2_half) >> __half_bits; // 64-bit
return __p3 + (__p1 >> __half_bits) + (__p2 >> __half_bits) + __carry;
}
_CCCL_TEMPLATE(typename _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]]
_CCCL_API constexpr _Tp mul_hi(_Tp __lhs, _Tp __rhs) noexcept
{
using ::cuda::std::int64_t;
using ::cuda::std::is_signed_v;
#if !_CCCL_TILE_COMPILATION() // nvbug6085239 error: calling a __device__ function from a __tile__ function
_CCCL_IF_NOT_CONSTEVAL_DEFAULT
{
if constexpr (sizeof(_Tp) == sizeof(int))
{
if constexpr (is_signed_v<_Tp>)
{
[[maybe_unused]] const auto __lhs1 = static_cast<int>(__lhs);
[[maybe_unused]] const auto __rhs1 = static_cast<int>(__rhs);
NV_IF_TARGET(NV_IS_DEVICE, (return ::__mulhi(__lhs1, __rhs1);));
}
else // is_unsigned_v<_Tp>
{
[[maybe_unused]] const auto __lhs1 = static_cast<unsigned>(__lhs);
[[maybe_unused]] const auto __rhs1 = static_cast<unsigned>(__rhs);
NV_IF_TARGET(NV_IS_DEVICE, (return ::__umulhi(__lhs1, __rhs1);));
}
}
else if constexpr (sizeof(_Tp) == sizeof(int64_t))
{
if constexpr (is_signed_v<_Tp>)
{
[[maybe_unused]] const auto __lhs1 = static_cast<long long>(__lhs);
[[maybe_unused]] const auto __rhs1 = static_cast<long long>(__rhs);
NV_IF_TARGET(NV_IS_DEVICE, (return ::__mul64hi(__lhs1, __rhs1);));
# if _CCCL_COMPILER(MSVC)
NV_IF_TARGET(NV_IS_HOST, (return ::__mulh(__lhs1, __rhs1);));
# endif // _CCCL_COMPILER(MSVC)
}
else // is_unsigned_v<_Tp>
{
[[maybe_unused]] const auto __lhs1 = static_cast<unsigned long long>(__lhs);
[[maybe_unused]] const auto __rhs1 = static_cast<unsigned long long>(__rhs);
NV_IF_TARGET(NV_IS_DEVICE, (return ::__umul64hi(__lhs1, __rhs1);));
# if _CCCL_COMPILER(MSVC)
NV_IF_TARGET(NV_IS_HOST, (return ::__umulh(__lhs1, __rhs1);));
# endif // _CCCL_COMPILER(MSVC)
}
}
}
#endif // !_CCCL_TILE_COMPILATION()
if constexpr (sizeof(_Tp) < sizeof(int64_t) || (sizeof(_Tp) == sizeof(int64_t) && _CCCL_HAS_INT128()))
{
constexpr auto __bits = ::cuda::std::__num_bits_v<_Tp>;
using __larger_t = ::cuda::std::__make_nbit_int_t<__bits * 2, is_signed_v<_Tp>>;
const auto __ret = (static_cast<__larger_t>(__lhs) * __rhs) >> __bits;
return static_cast<_Tp>(__ret);
}
else // sizeof(_Tp) >= sizeof(int64_t) && !_CCCL_HAS_INT128()
{
if constexpr (is_signed_v<_Tp>)
{
using _Up = ::cuda::std::make_unsigned_t<_Tp>;
const auto __lhs1 = static_cast<_Up>(__lhs);
const auto __rhs1 = static_cast<_Up>(__rhs);
auto __hi = ::cuda::__mul_hi_fallback(__lhs1, __rhs1);
if (__lhs < 0)
{
__hi -= __rhs1;
}
if (__rhs < 0)
{
__hi -= __lhs1;
}
return static_cast<_Tp>(__hi);
}
else
{
return ::cuda::__mul_hi_fallback(__lhs, __rhs);
}
}
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___CMATH_MULTIPLY_HIGH_HALF_H

47
qwen3_6_scripts/cccl_preload/include/cuda/__cmath/neg.h
View File

@@ -1,47 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___CMATH_NEG_H
#define _CUDA___CMATH_NEG_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/make_unsigned.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @brief Returns the negative value of the input number
//! @param __v The input number
//! @return The signed negative value of \p __v
//! @note This function doesn't cause undefined behavior when negating the minimum value of a signed integer type.
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr _Tp neg(_Tp __v) noexcept
{
return static_cast<_Tp>(~::cuda::std::__to_unsigned_like(__v) + 1);
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___CMATH_NEG_H

74
qwen3_6_scripts/cccl_preload/include/cuda/__cmath/pow2.h
View File

@@ -1,74 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___CMATH_POW2_H
#define _CUDA___CMATH_POW2_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__bit/has_single_bit.h>
#include <cuda/std/__bit/integral.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__type_traits/make_unsigned.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
_CCCL_TEMPLATE(typename _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr bool is_power_of_two(_Tp __t) noexcept
{
if constexpr (::cuda::std::is_signed_v<_Tp>)
{
_CCCL_ASSERT(__t >= _Tp{0}, "cuda::is_power_of_two requires non-negative input");
}
using _Up = ::cuda::std::make_unsigned_t<_Tp>;
return ::cuda::std::has_single_bit(static_cast<_Up>(__t));
}
_CCCL_TEMPLATE(typename _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr _Tp next_power_of_two(_Tp __t) noexcept
{
if constexpr (::cuda::std::is_signed_v<_Tp>)
{
_CCCL_ASSERT(__t >= _Tp{0}, "cuda::is_power_of_two requires non-negative input");
}
using _Up = ::cuda::std::make_unsigned_t<_Tp>;
return ::cuda::std::bit_ceil(static_cast<_Up>(__t));
}
_CCCL_TEMPLATE(typename _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr _Tp prev_power_of_two(_Tp __t) noexcept
{
if constexpr (::cuda::std::is_signed_v<_Tp>)
{
_CCCL_ASSERT(__t >= _Tp{0}, "cuda::is_power_of_two requires non-negative input");
}
using _Up = ::cuda::std::make_unsigned_t<_Tp>;
return ::cuda::std::bit_floor(static_cast<_Up>(__t));
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___CMATH_POW2_H

102
qwen3_6_scripts/cccl_preload/include/cuda/__cmath/round_down.h
View File

@@ -1,102 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___CMATH_ROUND_DOWN_H
#define _CUDA___CMATH_ROUND_DOWN_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__type_traits/common_type.h>
#include <cuda/std/__type_traits/is_enum.h>
#include <cuda/std/__type_traits/is_integral.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__type_traits/make_unsigned.h>
#include <cuda/std/__utility/to_underlying.h>
#include <cuda/std/limits>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @brief Round the number \p __a to the previous multiple of \p __b
//! @param __a The input number
//! @param __b The multiplicand
//! @pre \p __a must be non-negative
//! @pre \p __b must be positive
_CCCL_TEMPLATE(class _Tp, class _Up)
_CCCL_REQUIRES(::cuda::std::is_integral_v<_Tp> _CCCL_AND ::cuda::std::is_integral_v<_Up>)
[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<_Tp, _Up> round_down(const _Tp __a, const _Up __b) noexcept
{
_CCCL_ASSERT(__b > _Up{0}, "cuda::round_down: 'b' must be positive");
if constexpr (::cuda::std::is_signed_v<_Tp>)
{
_CCCL_ASSERT(__a >= _Tp{0}, "cuda::round_down: 'a' must be non negative");
}
using _Common = ::cuda::std::common_type_t<_Tp, _Up>;
using _Prom = decltype(_Tp{} / _Up{});
using _UProm = ::cuda::std::make_unsigned_t<_Prom>;
auto __c1 = static_cast<_UProm>(__a) / static_cast<_UProm>(__b);
return static_cast<_Common>(__c1 * static_cast<_UProm>(__b));
}
//! @brief Round the number \p __a to the previous multiple of \p __b
//! @param __a The input number
//! @param __b The multiplicand
//! @pre \p __a must be non-negative
//! @pre \p __b must be positive
_CCCL_TEMPLATE(class _Tp, class _Up)
_CCCL_REQUIRES(::cuda::std::is_integral_v<_Tp> _CCCL_AND ::cuda::std::is_enum_v<_Up>)
[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<_Tp, ::cuda::std::underlying_type_t<_Up>>
round_down(const _Tp __a, const _Up __b) noexcept
{
return ::cuda::round_down(__a, ::cuda::std::to_underlying(__b));
}
//! @brief Round the number \p __a to the previous multiple of \p __b
//! @param __a The input number
//! @param __b The multiplicand
//! @pre \p __a must be non-negative
//! @pre \p __b must be positive
_CCCL_TEMPLATE(class _Tp, class _Up)
_CCCL_REQUIRES(::cuda::std::is_enum_v<_Tp> _CCCL_AND ::cuda::std::is_integral_v<_Up>)
[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<::cuda::std::underlying_type_t<_Tp>, _Up>
round_down(const _Tp __a, const _Up __b) noexcept
{
return ::cuda::round_down(::cuda::std::to_underlying(__a), __b);
}
//! @brief Round the number \p __a to the previous multiple of \p __b
//! @param __a The input number
//! @param __b The multiplicand
//! @pre \p __a must be non-negative
//! @pre \p __b must be positive
_CCCL_TEMPLATE(class _Tp, class _Up)
_CCCL_REQUIRES(::cuda::std::is_enum_v<_Tp> _CCCL_AND ::cuda::std::is_enum_v<_Up>)
[[nodiscard]]
_CCCL_API constexpr ::cuda::std::common_type_t<::cuda::std::underlying_type_t<_Tp>, ::cuda::std::underlying_type_t<_Up>>
round_down(const _Tp __a, const _Up __b) noexcept
{
return ::cuda::round_down(::cuda::std::to_underlying(__a), ::cuda::std::to_underlying(__b));
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___CMATH_ROUND_DOWN_H

57
qwen3_6_scripts/cccl_preload/include/cuda/__cmath/uabs.h
View File

@@ -1,57 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___CMATH_UABS_H
#define _CUDA___CMATH_UABS_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__cmath/neg.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__type_traits/make_unsigned.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @brief Returns the *unsigned* absolute value of the given number.
//! @param __v The input number
//! @pre \p __v must be an integer type
//! @return The unsigned absolute value of \p __v
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_cv_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr ::cuda::std::make_unsigned_t<_Tp> uabs(_Tp __v) noexcept
{
if constexpr (::cuda::std::is_signed_v<_Tp>)
{
using _Up = ::cuda::std::make_unsigned_t<_Tp>;
return (__v < _Tp(0)) ? static_cast<_Up>(::cuda::neg(__v)) : static_cast<_Up>(__v);
}
else
{
return __v;
}
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___CMATH_UABS_H

181
qwen3_6_scripts/cccl_preload/include/cuda/__device/device_ref.h
View File

@@ -1,181 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___DEVICE_DEVICE_REF_H
#define _CUDA___DEVICE_DEVICE_REF_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
# include <cuda/__driver/driver_api.h>
# include <cuda/__fwd/devices.h>
# include <cuda/__runtime/types.h>
# include <cuda/std/span>
# include <cuda/std/string_view>
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
_CCCL_DIAG_PUSH
_CCCL_DIAG_SUPPRESS_CLANG("-Wmissing-braces")
// clang complains about missing braces in CUmemLocation constructor but GCC complains if we add them
::cuda::std::size_t __physical_devices_count();
//! @brief A non-owning representation of a CUDA device
class device_ref
{
int __id_ = 0;
public:
//! @brief Create a `device_ref` object from a native device ordinal.
/*implicit*/ _CCCL_HOST_API constexpr device_ref(int __id)
: __id_(__id)
{
_CCCL_IF_CONSTEVAL_DEFAULT
{
_CCCL_VERIFY(__id >= 0, "Device ID must be a valid GPU device ordinal");
}
else
{
_CCCL_VERIFY(__id >= 0 && static_cast<::cuda::std::size_t>(__id) < ::cuda::__physical_devices_count(),
"Device ID must be a valid GPU device ordinal");
}
}
//! @brief Retrieve the native ordinal of the `device_ref`
//!
//! @return int The native device ordinal held by the `device_ref` object
[[nodiscard]] _CCCL_HOST_API constexpr int get() const noexcept
{
return __id_;
}
//! @brief Compares two `device_ref`s for equality
//!
//! @note Allows comparison with `int` due to implicit conversion to
//! `device_ref`.
//!
//! @param __lhs The first `device_ref` to compare
//! @param __rhs The second `device_ref` to compare
//! @return true if `lhs` and `rhs` refer to the same device ordinal
[[nodiscard]] friend _CCCL_HOST_API constexpr bool operator==(device_ref __lhs, device_ref __rhs) noexcept
{
return __lhs.__id_ == __rhs.__id_;
}
# if _CCCL_STD_VER <= 2017
//! @brief Compares two `device_ref`s for inequality
//!
//! @note Allows comparison with `int` due to implicit conversion to
//! `device_ref`.
//!
//! @param __lhs The first `device_ref` to compare
//! @param __rhs The second `device_ref` to compare
//! @return true if `lhs` and `rhs` refer to different device ordinal
[[nodiscard]] friend _CCCL_HOST_API constexpr bool operator!=(device_ref __lhs, device_ref __rhs) noexcept
{
return __lhs.__id_ != __rhs.__id_;
}
# endif // _CCCL_STD_VER <= 2017
//! @brief Retrieve the specified attribute for the device
//!
//! @param __attr The attribute to query. See `device::attrs` for the available
//! attributes.
//!
//! @throws cuda_error if the attribute query fails
//!
//! @sa device::attrs
template <typename _Attr>
[[nodiscard]] _CCCL_HOST_API auto attribute(_Attr __attr) const
{
return __attr(*this);
}
//! @overload
template <::cudaDeviceAttr _Attr>
[[nodiscard]] _CCCL_HOST_API auto attribute() const
{
return attribute(__dev_attr<_Attr>());
}
//! @brief Retrieve the memory location of this device
//!
//! @return The memory location of this device
[[nodiscard]] _CCCL_HOST_API operator memory_location() const noexcept
{
return memory_location{::cudaMemLocationTypeDevice, get()};
}
//! @brief Initializes the primary context of the device.
_CCCL_HOST_API void init() const; // implemented in <cuda/__device/physical_device.h> to avoid circular dependency
//! @brief Retrieve the primary context of this device.
//!
//! @return The primary CUDA context for this device.
[[nodiscard]] _CCCL_HOST_API ::CUcontext __primary_context() const; // implemented in
// <cuda/__device/physical_device.h> to avoid
// circular dependency
//! @brief Retrieve the name of this device.
//!
//! @return String view containing the name of this device.
[[nodiscard]] _CCCL_HOST_API ::cuda::std::string_view name() const; // implemented in
// <cuda/__device/physical_device.h> to avoid
// circular dependency
//! @brief Queries if its possible for this device to directly access specified device's memory.
//!
//! If this function returns true, device supplied to this call can be passed into enable_peer_access
//! on memory resource or pool that manages memory on this device. It will make allocations from that
//! pool accessible by this device.
//!
//! @param __other_dev Device to query the peer access
//! @return true if its possible for this device to access the specified device's memory
[[nodiscard]] _CCCL_HOST_API bool has_peer_access_to(device_ref __other_dev) const
{
return ::cuda::__driver::__deviceCanAccessPeer(
::cuda::__driver::__deviceGet(get()), ::cuda::__driver::__deviceGet(__other_dev.get()));
}
// TODO this might return some more complex type in the future
// TODO we might want to include the calling device, depends on what we decide
// peer access APIs
//! @brief Retrieve `device_ref`s that are peers of this device
//!
//! The device on which this API is called is not included in the vector.
//!
//! @throws cuda_error if any peer access query fails
[[nodiscard]] _CCCL_HOST_API ::cuda::std::span<const device_ref> peers() const; // implemented in
// <cuda/__device/physical_device.h>
// to avoid circular dependency
};
_CCCL_DIAG_POP
_CCCL_END_NAMESPACE_CUDA
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
#endif // _CUDA___DEVICE_DEVICE_REF_H

227
qwen3_6_scripts/cccl_preload/include/cuda/__device/physical_device.h
View File

@@ -1,227 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___DEVICE_PHYSICAL_DEVICE_H
#define _CUDA___DEVICE_PHYSICAL_DEVICE_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
# include <cuda/__device/device_ref.h>
# include <cuda/__driver/driver_api.h>
# include <cuda/__fwd/devices.h>
# include <cuda/std/__cstddef/types.h>
# include <cuda/std/__memory/unique_ptr.h>
# include <cuda/std/cassert>
# include <cuda/std/span>
# include <cuda/std/string_view>
# if _CCCL_HOSTED()
# include <mutex>
# endif // _CCCL_HOSTED()
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
[[nodiscard]] inline ::cuda::std::span<__physical_device> __physical_devices();
// This is the element type of the the global `devices` array. In the future, we
// can cache device properties here.
//
//! @brief An immovable "owning" representation of a CUDA device.
class __physical_device
{
friend _CCCL_HOST_API inline ::cuda::std::unique_ptr<__physical_device[]>
__make_physical_devices(::cuda::std::size_t __device_count);
::CUdevice __device_{};
# if _CCCL_HOSTED()
::std::once_flag __primary_ctx_once_flag_{};
# endif // _CCCL_HOSTED()
::CUcontext __primary_ctx_{};
static constexpr ::cuda::std::size_t __max_name_length{256};
# if _CCCL_HOSTED()
::std::once_flag __name_once_flag_{};
# endif // _CCCL_HOSTED()
char __name_[__max_name_length]{};
::cuda::std::size_t __name_length_{};
# if _CCCL_HOSTED()
::std::once_flag __peers_once_flag_{};
# endif // _CCCL_HOSTED()
::cuda::std::unique_ptr<device_ref[]> __peers_{};
::cuda::std::size_t __num_peers_{};
_CCCL_HOST_API void __set_name()
{
const auto __id = ::cuda::__driver::__cudevice_to_ordinal(__device_);
::cuda::__driver::__deviceGetName(__name_, __max_name_length, __id);
__name_length_ = ::cuda::std::char_traits<char>::length(__name_);
}
_CCCL_HOST_API void __set_peers()
{
const auto __count = static_cast<int>(::cuda::__physical_devices().size());
const auto __id = ::cuda::__driver::__cudevice_to_ordinal(__device_);
// This overallocates, but given that we are talking about `device_ref` this is fine
__peers_.reset(static_cast<device_ref*>(::operator new[](sizeof(device_ref) * __count)));
size_t __num_peers = 0;
for (int __other_id = 0; __other_id < __count; ++__other_id)
{
// Exclude the device this API is called on. The main use case for this API
// is enable/disable peer access. While enable peer access can be called on
// device on which memory resides, disable peer access will error-out.
// Usage of the peer access control is smoother when *this is excluded,
// while it can be easily added with .push_back() on the vector if a full
// group of peers is needed (for cases other than peer access control)
if (__other_id != __id)
{
device_ref __dev{__id};
device_ref __other_dev{__other_id};
// While in almost all practical applications peer access should be symmetrical,
// it is possible to build a system with one directional peer access, check
// both ways here just to be safe
if (__dev.has_peer_access_to(__other_dev) && __other_dev.has_peer_access_to(__dev))
{
__peers_[__num_peers] = __other_dev;
++__num_peers;
}
}
}
__num_peers_ = __num_peers;
}
public:
_CCCL_HIDE_FROM_ABI __physical_device() = default;
_CCCL_HOST_API ~__physical_device()
{
if (__primary_ctx_ != nullptr)
{
[[maybe_unused]] const auto __ignore = ::cuda::__driver::__primaryCtxReleaseNoThrow(__device_);
}
}
//! @brief Retrieve the primary context for this device.
//!
//! @return A reference to the primary context for this device.
[[nodiscard]] _CCCL_HOST_API ::CUcontext __primary_context()
{
# if _CCCL_HOSTED()
::std::call_once(__primary_ctx_once_flag_, [this]() {
__primary_ctx_ = ::cuda::__driver::__primaryCtxRetain(__device_);
});
# else // ^^^ _CCCL_HOSTED() ^^^ / vvv _CCCL_FREESTANDING() vvv
if (!__primary_ctx_)
{
__primary_ctx_ = ::cuda::__driver::__primaryCtxRetain(__device_);
}
# endif // _CCCL_FREESTANDING()
return __primary_ctx_;
}
[[nodiscard]] _CCCL_HOST_API ::cuda::std::string_view __name()
{
# if _CCCL_HOSTED()
::std::call_once(__name_once_flag_, [this]() {
this->__set_name();
});
# else // ^^^ _CCCL_HOSTED() ^^^ / vvv _CCCL_FREESTANDING() vvv
if (__name_length_ != 0)
{
this->__set_name();
}
# endif // _CCCL_FREESTANDING()
return ::cuda::std::string_view{__name_, __name_length_};
}
[[nodiscard]] _CCCL_HOST_API ::cuda::std::span<const device_ref> __peers()
{
# if _CCCL_HOSTED()
::std::call_once(__peers_once_flag_, [this]() {
this->__set_peers();
});
# else // ^^^ _CCCL_HOSTED() ^^^ / vvv _CCCL_FREESTANDING() vvv
if (!__peers_)
{
this->__set_peers();
}
# endif // _CCCL_FREESTANDING()
return ::cuda::std::span<const device_ref>{__peers_.get(), __num_peers_};
}
};
[[nodiscard]] _CCCL_HOST_API inline ::cuda::std::unique_ptr<__physical_device[]>
__make_physical_devices(::cuda::std::size_t __device_count)
{
::cuda::std::unique_ptr<__physical_device[]> __devices{::new __physical_device[__device_count]};
for (::cuda::std::size_t __i = 0; __i < __device_count; ++__i)
{
__devices[__i].__device_ = static_cast<int>(__i);
}
return __devices;
}
[[nodiscard]] inline ::cuda::std::size_t __physical_devices_count()
{
static const auto __device_count = static_cast<::cuda::std::size_t>(::cuda::__driver::__deviceGetCount());
return __device_count;
}
[[nodiscard]] inline ::cuda::std::span<__physical_device> __physical_devices()
{
static const auto __device_count = __physical_devices_count();
static const auto __devices = ::cuda::__make_physical_devices(__device_count);
return ::cuda::std::span<__physical_device>{__devices.get(), __device_count};
}
// device_ref methods dependent on __physical_device
_CCCL_HOST_API inline void device_ref::init() const
{
(void) ::cuda::__physical_devices()[__id_].__primary_context();
}
[[nodiscard]] _CCCL_HOST_API inline ::CUcontext device_ref::__primary_context() const
{
return ::cuda::__physical_devices()[__id_].__primary_context();
}
[[nodiscard]] _CCCL_HOST_API inline ::cuda::std::string_view device_ref::name() const
{
return ::cuda::__physical_devices()[__id_].__name();
}
[[nodiscard]] _CCCL_HOST_API inline ::cuda::std::span<const device_ref> device_ref::peers() const
{
return ::cuda::__physical_devices()[__id_].__peers();
}
_CCCL_END_NAMESPACE_CUDA
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
#endif // _CUDA___DEVICE_PHYSICAL_DEVICE_H

1150
qwen3_6_scripts/cccl_preload/include/cuda/__driver/driver_api.h
View File

File diff suppressed because it is too large Load Diff

171
qwen3_6_scripts/cccl_preload/include/cuda/__event/event.h
View File

@@ -1,171 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___EVENT_EVENT_H
#define _CUDA___EVENT_EVENT_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
# include <cuda/__device/device_ref.h>
# include <cuda/__driver/driver_api.h>
# include <cuda/__event/event_ref.h>
# include <cuda/__runtime/ensure_current_context.h>
# include <cuda/__utility/no_init.h>
# include <cuda/std/__utility/to_underlying.h>
# include <cuda/std/cstddef>
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
class timed_event;
//! @brief Flags to use when creating the event.
enum class event_flags : unsigned
{
none = cudaEventDefault,
blocking_sync = cudaEventBlockingSync,
interprocess = cudaEventInterprocess,
};
[[nodiscard]] _CCCL_HOST_API constexpr event_flags operator|(event_flags __lhs, event_flags __rhs) noexcept
{
return static_cast<event_flags>(::cuda::std::to_underlying(__lhs) | ::cuda::std::to_underlying(__rhs));
}
//! @brief An owning wrapper for an untimed `cudaEvent_t`.
class event : public event_ref
{
friend class timed_event;
public:
//! @brief Construct a new `event` object with timing disabled, and record
//! the event in the specified stream.
//!
//! @throws cuda_error if the event creation fails.
_CCCL_HOST_API explicit event(stream_ref __stream, event_flags __flags = event_flags::none);
//! @brief Construct a new `event` object with timing disabled. The event can only be recorded on streams from the
//! specified device.
//!
//! @throws cuda_error if the event creation fails.
_CCCL_HOST_API explicit event(device_ref __device, event_flags __flags = event_flags::none)
: event(__device, ::cuda::std::to_underlying(__flags) | cudaEventDisableTiming)
{}
//! @brief Construct a new `event` object into the moved-from state.
//!
//! @post `get()` returns `cudaEvent_t()`.
_CCCL_HOST_API explicit constexpr event(no_init_t) noexcept
: event_ref(::cudaEvent_t{})
{}
//! @brief Move-construct a new `event` object
//!
//! @param __other
//!
//! @post `__other` is in a moved-from state.
_CCCL_HOST_API constexpr event(event&& __other) noexcept
: event_ref(::cuda::std::exchange(__other.__event_, {}))
{}
// Disallow copy construction.
event(const event&) = delete;
//! @brief Destroy the `event` object
//!
//! @note If the event fails to be destroyed, the error is silently ignored.
_CCCL_HOST_API ~event()
{
if (__event_ != nullptr)
{
// Needs to call driver API in case current device is not set, runtime version would set dev 0 current
// Alternative would be to store the device and push/pop here
[[maybe_unused]] auto __status = ::cuda::__driver::__eventDestroyNoThrow(__event_);
}
}
//! @brief Move-assign an `event` object
//!
//! @param __other
//!
//! @post `__other` is in a moved-from state.
_CCCL_HOST_API event& operator=(event&& __other) noexcept
{
event __tmp(::cuda::std::move(__other));
::cuda::std::swap(__event_, __tmp.__event_);
return *this;
}
// Disallow copy assignment.
event& operator=(const event&) = delete;
//! @brief Construct an `event` object from a native `cudaEvent_t` handle.
//!
//! @param __evnt The native handle
//!
//! @return event The constructed `event` object
//!
//! @note The constructed `event` object takes ownership of the native handle.
[[nodiscard]] static _CCCL_HOST_API event from_native_handle(::cudaEvent_t __evnt) noexcept
{
return event(__evnt);
}
// Disallow construction from an `int`, e.g., `0`.
static event from_native_handle(int) = delete;
// Disallow construction from `nullptr`.
static event from_native_handle(::cuda::std::nullptr_t) = delete;
//! @brief Retrieve the native `cudaEvent_t` handle and give up ownership.
//!
//! @return cudaEvent_t The native handle being held by the `event` object.
//!
//! @post The event object is in a moved-from state.
[[nodiscard]] _CCCL_HOST_API constexpr ::cudaEvent_t release() noexcept
{
return ::cuda::std::exchange(__event_, {});
}
private:
// Use `event::from_native_handle(e)` to construct an owning `event`
// object from a `cudaEvent_t` handle.
_CCCL_HOST_API explicit constexpr event(::cudaEvent_t __evnt) noexcept
: event_ref(__evnt)
{}
_CCCL_HOST_API explicit event(stream_ref __stream, unsigned __flags);
_CCCL_HOST_API explicit event(device_ref __device, unsigned __flags)
: event_ref(::cudaEvent_t{})
{
[[maybe_unused]] __ensure_current_context __ctx_setter(__device);
__event_ = ::cuda::__driver::__eventCreate(static_cast<unsigned>(__flags));
}
};
_CCCL_END_NAMESPACE_CUDA
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
#endif // _CUDA___EVENT_EVENT_H

158
qwen3_6_scripts/cccl_preload/include/cuda/__event/event_ref.h
View File

@@ -1,158 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___EVENT_EVENT_REF_H
#define _CUDA___EVENT_EVENT_REF_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
# include <cuda/__driver/driver_api.h>
# include <cuda/std/__exception/cuda_error.h>
# include <cuda/std/__exception/exception_macros.h>
# include <cuda/std/cassert>
# include <cuda/std/cstddef>
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
class event;
class timed_event;
class stream_ref;
//! @brief An non-owning wrapper for an untimed `cudaEvent_t`.
class event_ref
{
private:
friend class event;
friend class timed_event;
::cudaEvent_t __event_{};
public:
using value_type = ::cudaEvent_t;
//! @brief Construct a new `event_ref` object from a `cudaEvent_t`
//!
//! This constructor provides an implicit conversion from `cudaEvent_t`
//!
//! @post `get() == __evnt`
//!
//! @note: It is the callers responsibility to ensure the `event_ref` does not
//! outlive the event denoted by the `cudaEvent_t` handle.
_CCCL_HOST_API constexpr event_ref(::cudaEvent_t __evnt) noexcept
: __event_(__evnt)
{}
/// Disallow construction from an `int`, e.g., `0`.
event_ref(int) = delete;
/// Disallow construction from `nullptr`.
event_ref(::cuda::std::nullptr_t) = delete;
//! @brief Records an event on the specified stream
//!
//! @param __stream
//!
//! @throws cuda_error if the event record fails
_CCCL_HOST_API void record(stream_ref __stream) const;
//! @brief Synchronizes the event
//!
//! @throws cuda_error if waiting for the event fails
_CCCL_HOST_API void sync() const
{
_CCCL_ASSERT(__event_ != nullptr, "cuda::event_ref::sync no event set");
::cuda::__driver::__eventSynchronize(__event_);
}
//! @brief Checks if all the work in the stream prior to the record of the event has completed.
//!
//! If is_done returns true, calling sync() on this event will return immediately
//!
//! @throws cuda_error if the event query fails
[[nodiscard]] _CCCL_HOST_API bool is_done() const
{
_CCCL_ASSERT(__event_ != nullptr, "cuda::event_ref::sync no event set");
::cudaError_t __status = ::cuda::__driver::__eventQueryNoThrow(__event_);
if (__status == ::cudaSuccess)
{
return true;
}
else if (__status == ::cudaErrorNotReady)
{
return false;
}
else
{
_CCCL_THROW(::cuda::cuda_error, __status, "Failed to query CUDA event");
}
}
//! @brief Retrieve the native `cudaEvent_t` handle.
//!
//! @return cudaEvent_t The native handle being held by the event_ref object.
[[nodiscard]] _CCCL_HOST_API constexpr ::cudaEvent_t get() const noexcept
{
return __event_;
}
//! @brief Checks if the `event_ref` is valid
//!
//! @return true if the `event_ref` is valid, false otherwise.
[[nodiscard]] _CCCL_HOST_API explicit constexpr operator bool() const noexcept
{
return __event_ != nullptr;
}
//! @brief Compares two `event_ref`s for equality
//!
//! @note Allows comparison with `cudaEvent_t` due to implicit conversion to
//! `event_ref`.
//!
//! @param __lhs The first `event_ref` to compare
//! @param __rhs The second `event_ref` to compare
//! @return true if `lhs` and `rhs` refer to the same `cudaEvent_t` object.
[[nodiscard]] friend _CCCL_HOST_API constexpr bool operator==(event_ref __lhs, event_ref __rhs) noexcept
{
return __lhs.__event_ == __rhs.__event_;
}
//! @brief Compares two `event_ref`s for inequality
//!
//! @note Allows comparison with `cudaEvent_t` due to implicit conversion to
//! `event_ref`.
//!
//! @param __lhs The first `event_ref` to compare
//! @param __rhs The second `event_ref` to compare
//! @return true if `lhs` and `rhs` refer to different `cudaEvent_t` objects.
[[nodiscard]] friend _CCCL_HOST_API constexpr bool operator!=(event_ref __lhs, event_ref __rhs) noexcept
{
return __lhs.__event_ != __rhs.__event_;
}
};
_CCCL_END_NAMESPACE_CUDA
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
#endif // _CUDA___EVENT_EVENT_REF_H

117
qwen3_6_scripts/cccl_preload/include/cuda/__event/timed_event.h
View File

@@ -1,117 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___EVENT_TIMED_EVENT_H
#define _CUDA___EVENT_TIMED_EVENT_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
# include <cuda/__device/device_ref.h>
# include <cuda/__driver/driver_api.h>
# include <cuda/__event/event.h>
# include <cuda/__utility/no_init.h>
# include <cuda/std/__chrono/duration.h>
# include <cuda/std/__utility/to_underlying.h>
# include <cuda/std/cstddef>
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @brief An owning wrapper for a `cudaEvent_t` with timing enabled.
class timed_event : public event
{
public:
//! @brief Construct a new `timed_event` object with the specified flags
//! and record the event on the specified stream.
//!
//! @throws cuda_error if the event creation fails.
_CCCL_HOST_API explicit timed_event(stream_ref __stream, event_flags __flags = event_flags::none);
//! @brief Construct a new `timed_event` object with the specified flags. The event can only be recorded on streams
//! from the specified device.
//!
//! @throws cuda_error if the event creation fails.
_CCCL_HOST_API explicit timed_event(device_ref __device, event_flags __flags = event_flags::none)
: event(__device, ::cuda::std::to_underlying(__flags))
{}
//! @brief Construct a new `timed_event` object into the moved-from state.
//!
//! @post `get()` returns `cudaEvent_t()`.
_CCCL_HOST_API explicit constexpr timed_event(no_init_t) noexcept
: event(no_init)
{}
timed_event(timed_event&&) noexcept = default;
timed_event(const timed_event&) = delete;
timed_event& operator=(timed_event&&) noexcept = default;
timed_event& operator=(const timed_event&) = delete;
//! @brief Construct a `timed_event` object from a native `cudaEvent_t` handle.
//!
//! @param __evnt The native handle
//!
//! @return timed_event The constructed `timed_event` object
//!
//! @note The constructed `timed_event` object takes ownership of the native handle.
[[nodiscard]] static _CCCL_HOST_API timed_event from_native_handle(::cudaEvent_t __evnt) noexcept
{
return timed_event(__evnt);
}
// Disallow construction from an `int`, e.g., `0`.
static timed_event from_native_handle(int) = delete;
// Disallow construction from `nullptr`.
static timed_event from_native_handle(::cuda::std::nullptr_t) = delete;
//! @brief Compute the time elapsed between two `timed_event` objects.
//!
//! @throws cuda_error if the query for the elapsed time fails.
//!
//! @param __end The `timed_event` object representing the end time.
//! @param __start The `timed_event` object representing the start time.
//!
//! @return cuda::std::chrono::nanoseconds The elapsed time in nanoseconds.
//!
//! @note The elapsed time has a resolution of approximately 0.5 microseconds.
[[nodiscard]] friend _CCCL_HOST_API ::cuda::std::chrono::nanoseconds
operator-(const timed_event& __end, const timed_event& __start)
{
const auto __ms = ::cuda::__driver::__eventElapsedTime(__start.get(), __end.get());
return ::cuda::std::chrono::nanoseconds(static_cast<::cuda::std::chrono::nanoseconds::rep>(__ms * 1'000'000.0));
}
private:
// Use `timed_event::from_native_handle(e)` to construct an owning `timed_event`
// object from a `cudaEvent_t` handle.
_CCCL_HOST_API explicit constexpr timed_event(::cudaEvent_t __evnt) noexcept
: event(__evnt)
{}
};
_CCCL_END_NAMESPACE_CUDA
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
#endif // _CUDA___EVENT_TIMED_EVENT_H

89
qwen3_6_scripts/cccl_preload/include/cuda/__execution/determinism.h
View File

@@ -1,89 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDA___EXECUTION_DETERMINISM_H
#define __CUDA___EXECUTION_DETERMINISM_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__execution/require.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__execution/env.h>
#include <cuda/std/__type_traits/is_one_of.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION
namespace determinism
{
struct __get_determinism_t;
enum class __determinism_t
{
__not_guaranteed,
__run_to_run,
__gpu_to_gpu
};
template <__determinism_t _Guarantee>
struct __determinism_holder_t : __requirement
{
static constexpr __determinism_t value = _Guarantee;
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto query(const __get_determinism_t&) const noexcept
-> __determinism_holder_t<_Guarantee>
{
return *this;
}
};
using gpu_to_gpu_t = __determinism_holder_t<__determinism_t::__gpu_to_gpu>;
using run_to_run_t = __determinism_holder_t<__determinism_t::__run_to_run>;
using not_guaranteed_t = __determinism_holder_t<__determinism_t::__not_guaranteed>;
_CCCL_GLOBAL_CONSTANT gpu_to_gpu_t gpu_to_gpu{};
_CCCL_GLOBAL_CONSTANT run_to_run_t run_to_run{};
_CCCL_GLOBAL_CONSTANT not_guaranteed_t not_guaranteed{};
struct __get_determinism_t
{
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Env)
_CCCL_REQUIRES(::cuda::std::execution::__queryable_with<_Env, __get_determinism_t>)
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto operator()(const _Env& __env) const noexcept
{
static_assert(noexcept(__env.query(*this)));
return __env.query(*this);
}
[[nodiscard]]
_CCCL_NODEBUG_API static constexpr auto query(::cuda::std::execution::forwarding_query_t) noexcept -> bool
{
return true;
}
};
_CCCL_GLOBAL_CONSTANT auto __get_determinism = __get_determinism_t{};
} // namespace determinism
_CCCL_END_NAMESPACE_CUDA_EXECUTION
#include <cuda/std/__cccl/epilogue.h>
#endif // __CUDA___EXECUTION_DETERMINISM_H

99
qwen3_6_scripts/cccl_preload/include/cuda/__execution/output_ordering.h
View File

@@ -1,99 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDA___EXECUTION_OUTPUT_ORDERING_H
#define __CUDA___EXECUTION_OUTPUT_ORDERING_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__execution/require.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__execution/env.h>
#include <cuda/std/__type_traits/integral_constant.h>
#include <cuda/std/__type_traits/is_one_of.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION
//! @brief Requirement describing the order in which an algorithm writes its results to the output (e.g. the top-k
//! elements of cub::DeviceBatchedTopK).
//!
//! The available options are:
//! - output_ordering::unsorted: the results may be written in any order.
//! - output_ordering::sorted: the results are written sorted by key. Among elements that compare
//! equal, their relative order is unspecified.
//! - output_ordering::stable_sorted: the results are written sorted by key and, among elements that
//! compare equal, their relative order matches the order of their source indices (smaller source index first).
namespace output_ordering
{
struct __get_output_ordering_t;
enum class __output_ordering_t
{
__sorted,
__unsorted,
__stable_sorted
};
template <__output_ordering_t _Guarantee>
struct _CCCL_DECLSPEC_EMPTY_BASES __output_ordering_holder_t
: __requirement
, ::cuda::std::integral_constant<__output_ordering_t, _Guarantee>
{
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto query(const __get_output_ordering_t&) const noexcept
-> __output_ordering_holder_t<_Guarantee>
{
return *this;
}
};
using sorted_t = __output_ordering_holder_t<__output_ordering_t::__sorted>;
using stable_sorted_t = __output_ordering_holder_t<__output_ordering_t::__stable_sorted>;
using unsorted_t = __output_ordering_holder_t<__output_ordering_t::__unsorted>;
_CCCL_GLOBAL_CONSTANT sorted_t sorted{};
_CCCL_GLOBAL_CONSTANT stable_sorted_t stable_sorted{};
_CCCL_GLOBAL_CONSTANT unsorted_t unsorted{};
struct __get_output_ordering_t
{
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Env)
_CCCL_REQUIRES(::cuda::std::execution::__queryable_with<_Env, __get_output_ordering_t>)
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto operator()(const _Env& __env) const noexcept
{
static_assert(noexcept(__env.query(*this)));
return __env.query(*this);
}
[[nodiscard]]
_CCCL_NODEBUG_API static constexpr auto query(::cuda::std::execution::forwarding_query_t) noexcept -> bool
{
return true;
}
};
_CCCL_GLOBAL_CONSTANT auto __get_output_ordering = __get_output_ordering_t{};
} // namespace output_ordering
_CCCL_END_NAMESPACE_CUDA_EXECUTION
#include <cuda/std/__cccl/epilogue.h>
#endif

75
qwen3_6_scripts/cccl_preload/include/cuda/__execution/require.h
View File

@@ -1,75 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDA___EXECUTION_REQUIRE_H
#define __CUDA___EXECUTION_REQUIRE_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__execution/env.h>
#include <cuda/std/__type_traits/is_base_of.h>
#include <cuda/std/__type_traits/is_empty.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION
class __requirement
{};
struct __get_requirements_t
{
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Env)
_CCCL_REQUIRES(::cuda::std::execution::__queryable_with<_Env, __get_requirements_t>)
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto operator()(const _Env& __env) const noexcept
{
static_assert(noexcept(__env.query(*this)));
return __env.query(*this);
}
[[nodiscard]]
_CCCL_NODEBUG_API static constexpr auto query(::cuda::std::execution::forwarding_query_t) noexcept -> bool
{
return true;
}
};
_CCCL_GLOBAL_CONSTANT auto __get_requirements = __get_requirements_t{};
template <class... _Requirements>
[[nodiscard]] _CCCL_NODEBUG_API auto require(_Requirements...)
{
static_assert((::cuda::std::is_base_of_v<__requirement, _Requirements> && ...),
"Only requirements can be passed to require");
static_assert((::cuda::std::is_empty_v<_Requirements> && ...), "Stateful requirements are not implemented");
// clang < 19 doesn't like this code
// since the only requirements we currently allow are in determinism.h and
// all of them are stateless, let's ignore incoming parameters
::cuda::std::execution::env<_Requirements...> __env{};
return ::cuda::std::execution::prop{__get_requirements_t{}, __env};
}
_CCCL_END_NAMESPACE_CUDA_EXECUTION
#include <cuda/std/__cccl/epilogue.h>
#endif // __CUDA___EXECUTION_REQUIRE_H

96
qwen3_6_scripts/cccl_preload/include/cuda/__execution/tie_break.h
View File

@@ -1,96 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDA___EXECUTION_TIE_BREAK_H
#define __CUDA___EXECUTION_TIE_BREAK_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__execution/require.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__execution/env.h>
#include <cuda/std/__type_traits/is_one_of.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION
//! @brief Requirement describing how an algorithm breaks ties among elements that compare equal at its selection
//! boundary (e.g. the K-th element of cub::DeviceBatchedTopK).
//!
//! A tie-break requirement only constrains *which* of the equal-comparing elements end up in the result set; it says
//! nothing about the order in which the results are written (that is controlled independently by
//! cuda::execution::output_ordering). A tie-break is only meaningful together with a deterministic execution
//! requirement (cuda::execution::determinism::run_to_run or gpu_to_gpu).
namespace tie_break
{
struct __get_tie_break_t;
enum class __tie_break_t
{
__unspecified, //!< Any (implementation-defined) deterministic tie-break is acceptable.
__prefer_smaller_index, //!< Among elements that compare equal, prefer the one(s) with the smaller source index.
__prefer_larger_index //!< Among elements that compare equal, prefer the one(s) with the larger source index.
};
template <__tie_break_t _Preference>
struct __tie_break_holder_t : __requirement
{
static constexpr __tie_break_t value = _Preference;
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto query(const __get_tie_break_t&) const noexcept
-> __tie_break_holder_t<_Preference>
{
return *this;
}
};
using unspecified_t = __tie_break_holder_t<__tie_break_t::__unspecified>;
using prefer_smaller_index_t = __tie_break_holder_t<__tie_break_t::__prefer_smaller_index>;
using prefer_larger_index_t = __tie_break_holder_t<__tie_break_t::__prefer_larger_index>;
_CCCL_GLOBAL_CONSTANT unspecified_t unspecified{};
_CCCL_GLOBAL_CONSTANT prefer_smaller_index_t prefer_smaller_index{};
_CCCL_GLOBAL_CONSTANT prefer_larger_index_t prefer_larger_index{};
struct __get_tie_break_t
{
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Env)
_CCCL_REQUIRES(::cuda::std::execution::__queryable_with<_Env, __get_tie_break_t>)
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto operator()(const _Env& __env) const noexcept
{
static_assert(noexcept(__env.query(*this)));
return __env.query(*this);
}
[[nodiscard]]
_CCCL_NODEBUG_API static constexpr auto query(::cuda::std::execution::forwarding_query_t) noexcept -> bool
{
return true;
}
};
_CCCL_GLOBAL_CONSTANT auto __get_tie_break = __get_tie_break_t{};
} // namespace tie_break
_CCCL_END_NAMESPACE_CUDA_EXECUTION
#include <cuda/std/__cccl/epilogue.h>
#endif // __CUDA___EXECUTION_TIE_BREAK_H

81
qwen3_6_scripts/cccl_preload/include/cuda/__execution/tune.h
View File

@@ -1,81 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDA___EXECUTION_TUNE_H
#define __CUDA___EXECUTION_TUNE_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__device/compute_capability.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__concepts/semiregular.h>
#include <cuda/std/__execution/env.h>
#include <cuda/std/__functional/invoke.h>
#include <cuda/std/__type_traits/is_empty.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION
struct __get_tuning_t
{
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Env)
_CCCL_REQUIRES(::cuda::std::execution::__queryable_with<_Env, __get_tuning_t>)
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto operator()(const _Env& __env) const noexcept
{
static_assert(noexcept(__env.query(*this)));
return __env.query(*this);
}
[[nodiscard]]
_CCCL_NODEBUG_API static constexpr auto query(::cuda::std::execution::forwarding_query_t) noexcept -> bool
{
return true;
}
};
_CCCL_GLOBAL_CONSTANT auto __get_tuning = __get_tuning_t{};
//! @rst
//! Creates an environment from a pack of policy selectors that can be passed to device-wide parallel algorithms to
//! select tunings for different target architectures. See the :ref:`policy selector documentation
//! <cub-policy-selectors>` for more information on how algorithms can be tuned.
//! @endrst
template <class... _PolicySelectors>
[[nodiscard]] _CCCL_NODEBUG_API auto tune(_PolicySelectors...)
{
static_assert((::cuda::std::is_empty_v<_PolicySelectors> && ...), "Policy selectors must be stateless");
static_assert((::cuda::std::semiregular<_PolicySelectors> && ...), "Policy selectors must be semiregular types");
static_assert((::cuda::std::is_invocable_v<_PolicySelectors, ::cuda::compute_capability> && ...),
"Policy selectors must be invocable with cuda::compute_capability");
// since all the tunings are stateless, let's ignore incoming parameters
// we use the return type of the policy_selector as tag
using tuning_env = ::cuda::std::execution::env<
::cuda::std::execution::prop<decltype(_PolicySelectors{}(::cuda::compute_capability{})), _PolicySelectors>...>;
return ::cuda::std::execution::prop{__get_tuning_t{}, tuning_env{}};
}
_CCCL_END_NAMESPACE_CUDA_EXECUTION
#include <cuda/std/__cccl/epilogue.h>
#endif // __CUDA___EXECUTION_TUNE_H

151
qwen3_6_scripts/cccl_preload/include/cuda/__functional/address_stability.h
View File

@@ -1,151 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___FUNCTIONAL_ADDRESS_STABILITY_H
#define _CUDA___FUNCTIONAL_ADDRESS_STABILITY_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__functional/always_true_false.h>
#include <cuda/__fwd/iterator.h>
#include <cuda/std/__functional/not_fn.h>
#include <cuda/std/__functional/operations.h>
#include <cuda/std/__functional/ranges_operations.h>
#include <cuda/std/__type_traits/integral_constant.h>
#include <cuda/std/__type_traits/is_class.h>
#include <cuda/std/__type_traits/is_enum.h>
#include <cuda/std/__type_traits/is_void.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! Trait telling whether a function object type F does not rely on the memory addresses of its arguments. The nested
//! value is true when the addresses of the arguments do not matter and arguments can be provided from arbitrary copies
//! of the respective sources. This trait can be specialized for custom function objects types.
//! @see proclaim_copyable_arguments
template <typename F, typename SFINAE = void>
struct proclaims_copyable_arguments : ::cuda::std::false_type
{};
template <typename F, typename... Args>
inline constexpr bool proclaims_copyable_arguments_v = proclaims_copyable_arguments<F, Args...>::value;
// Wrapper for a callable to mark it as permitting copied arguments
template <typename F>
struct __callable_permitting_copied_arguments : F
{
using F::operator();
};
template <typename F>
struct proclaims_copyable_arguments<__callable_permitting_copied_arguments<F>> : ::cuda::std::true_type
{};
//! Creates a new function object from an existing one, which is marked as permitting its arguments to be copies of
//! whatever source they come from. This implies that the addresses of the arguments are irrelevant to the function
//! object. Some algorithms, like thrust::transform, can benefit from this information and choose a more efficient
//! implementation.
//! @see proclaims_copyable_arguments
template <typename F>
[[nodiscard]] _CCCL_API constexpr auto proclaim_copyable_arguments(F&& f)
{
if constexpr (proclaims_copyable_arguments<F>::value)
{ // If F is already marked then we do not need to wrap it
return f;
}
else
{
return __callable_permitting_copied_arguments<::cuda::std::decay_t<F>>{::cuda::std::forward<F>(f)};
}
}
// Specializations for libcu++ function objects are provided here to not pull this include into `<cuda/std/...>` headers
template <typename _Fn>
struct proclaims_copyable_arguments<::cuda::std::__not_fn_t<_Fn>> : proclaims_copyable_arguments<_Fn>
{};
template <typename _Fn>
struct proclaims_copyable_arguments<zip_function<_Fn>> : proclaims_copyable_arguments<_Fn>
{};
template <typename _Tp>
struct __has_builtin_operators
: ::cuda::std::bool_constant<!::cuda::std::is_class_v<_Tp> && !::cuda::std::is_enum_v<_Tp>
&& !::cuda::std::is_void_v<_Tp>>
{};
#define _LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(functor) \
/*we know what plus<T> etc. does if T is not a type that could have a weird operatorX() */ \
template <typename _Tp> \
struct proclaims_copyable_arguments<functor<_Tp>> : ::cuda::__has_builtin_operators<_Tp> \
{}; \
/*we do not know what plus<void> etc. does, which depends on the types it is invoked on */ \
template <> \
struct proclaims_copyable_arguments<functor<void>> : ::cuda::std::false_type \
{};
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::plus)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::minus)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::multiplies)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::divides)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::modulus)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::negate)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::bit_and)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::bit_not)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::bit_or)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::bit_xor)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::equal_to)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::not_equal_to)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::less)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::less_equal)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::greater_equal)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::greater)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::logical_and)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::logical_not)
_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::logical_or)
#define _LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS(functor) \
/*we do not know what equal_to etc. does, which depends on the types and their operator== it is invoked on */ \
template <> \
struct proclaims_copyable_arguments<functor> : ::cuda::std::false_type \
{};
_LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS(::cuda::std::ranges::equal_to)
_LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS(::cuda::std::ranges::not_equal_to)
_LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS(::cuda::std::ranges::less)
_LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS(::cuda::std::ranges::less_equal)
_LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS(::cuda::std::ranges::greater)
_LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS(::cuda::std::ranges::greater_equal)
#undef _LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS
// always_true and always_false never inspect the addresses of their arguments
template <>
struct proclaims_copyable_arguments<::cuda::always_true> : ::cuda::std::true_type
{};
template <>
struct proclaims_copyable_arguments<::cuda::always_false> : ::cuda::std::true_type
{};
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___FUNCTIONAL_ADDRESS_STABILITY_H

52
qwen3_6_scripts/cccl_preload/include/cuda/__functional/always_true_false.h
View File

@@ -1,52 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___FUNCTIONAL_ALWAYS_TRUE_FALSE_H
#define _CUDA___FUNCTIONAL_ALWAYS_TRUE_FALSE_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @brief Function object that always returns \c true regardless of the arguments passed.
struct always_true
{
template <typename... _Ts>
[[nodiscard]] _CCCL_API constexpr bool operator()(_Ts&&...) const noexcept
{
return true;
}
};
//! @brief Function object that always returns \c false regardless of the arguments passed.
struct always_false
{
template <typename... _Ts>
[[nodiscard]] _CCCL_API constexpr bool operator()(_Ts&&...) const noexcept
{
return false;
}
};
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___FUNCTIONAL_ALWAYS_TRUE_FALSE_H

65
qwen3_6_scripts/cccl_preload/include/cuda/__functional/call_or.h
View File

@@ -1,65 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___FUNCTIONAL_CALL_OR_H
#define _CUDA___FUNCTIONAL_CALL_OR_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__tuple_dir/ignore.h>
#include <cuda/std/__type_traits/is_callable.h>
#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @brief `__call_or` is an higher-order function that accepts a function, a default
//! value, and arguments to call the function with. If the function is callable with the
//! provided arguments, it invokes the function and returns the result. Otherwise, it
//! returns the default value.
struct __call_or_t
{
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Fn, class _Fallback, class... _Args)
_CCCL_REQUIRES(::cuda::std::__is_callable_v<_Fn, _Args...>)
_CCCL_API constexpr auto operator()(_Fn __fn, _Fallback&&, _Args&&... __args) const
noexcept(::cuda::std::__is_nothrow_callable_v<_Fn, _Args...>) -> ::cuda::std::__call_result_t<_Fn, _Args...>
{
return __fn(static_cast<_Args&&>(__args)...);
}
_CCCL_EXEC_CHECK_DISABLE
template <class _Fallback, class... _Args>
_CCCL_API constexpr auto operator()(::cuda::std::__ignore_t, _Fallback&& __fallback, _Args&&...) const
noexcept(::cuda::std::is_nothrow_move_constructible_v<_Fallback>) -> _Fallback
{
return static_cast<_Fallback&&>(__fallback);
}
};
_CCCL_GLOBAL_CONSTANT auto __call_or = __call_or_t{};
template <class _Fn, class _Fallback, class... _Args>
using __call_result_or_t _CCCL_NODEBUG_ALIAS = ::cuda::std::__call_result_t<__call_or_t, _Fn, _Fallback, _Args...>;
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___FUNCTIONAL_CALL_OR_H

57
qwen3_6_scripts/cccl_preload/include/cuda/__functional/equal_to_value.h
View File

@@ -1,57 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___FUNCTIONAL_EQUAL_TO_VALUE_H
#define _CUDA___FUNCTIONAL_EQUAL_TO_VALUE_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__type_traits/is_comparable.h>
#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @brief `equal_to_value` is a function object that checks if a value is equal to a stored value.
//! @tparam _Tp The type of the value to be compared.
template <typename _Tp>
struct equal_to_value
{
_Tp __value_;
_CCCL_API explicit constexpr equal_to_value(const _Tp& __value) noexcept(
::cuda::std::is_nothrow_copy_constructible_v<_Tp>)
: __value_(__value)
{}
_CCCL_TEMPLATE(class _Up)
_CCCL_REQUIRES(::cuda::std::__is_cpp17_equality_comparable_v<_Tp, _Up>)
[[nodiscard]] _CCCL_API constexpr bool operator()(const _Up& __lhs) const
noexcept(::cuda::std::__is_cpp17_nothrow_equality_comparable_v<_Tp, _Up>)
{
return static_cast<bool>(__lhs == __value_);
}
};
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___FUNCTIONAL_EQUAL_TO_VALUE_H

77
qwen3_6_scripts/cccl_preload/include/cuda/__functional/maximum.h
View File

@@ -1,77 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_FUNCTIONAL_MAXIMUM_H
#define _CUDA_FUNCTIONAL_MAXIMUM_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__functional/minimum_maximum_common.h>
#include <cuda/std/__cmath/min_max.h>
#include <cuda/std/__type_traits/common_type.h>
#include <cuda/std/__type_traits/is_extended_floating_point.h>
#include <cuda/std/__type_traits/is_floating_point.h>
#include <cuda/std/__utility/ctad_support.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <class _Tp = void>
struct _CCCL_TYPE_VISIBILITY_DEFAULT maximum
{
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr _Tp operator()(const _Tp& __lhs, const _Tp& __rhs) const
noexcept(__is_maximum_minimum_noexcept_v<_Tp, _Tp, _Tp>)
{
if constexpr (::cuda::std::is_floating_point_v<_Tp> || ::cuda::std::__is_extended_floating_point_v<_Tp>)
{
return ::cuda::std::fmax(__lhs, __rhs);
}
else
{
return (__lhs < __rhs) ? __rhs : __lhs;
}
}
};
_CCCL_CTAD_SUPPORTED_FOR_TYPE(maximum);
template <>
struct _CCCL_TYPE_VISIBILITY_DEFAULT maximum<void>
{
_CCCL_EXEC_CHECK_DISABLE
template <class Tp, class Up, class _Common = ::cuda::std::common_type_t<Tp, Up>>
[[nodiscard]] _CCCL_API constexpr _Common operator()(const Tp& __lhs, const Up& __rhs) const
noexcept(__is_maximum_minimum_noexcept_v<Tp, Up, _Common>)
{
if constexpr (::cuda::std::is_floating_point_v<_Common> || ::cuda::std::__is_extended_floating_point_v<_Common>)
{
return ::cuda::std::fmax(static_cast<_Common>(__lhs), static_cast<_Common>(__rhs));
}
else
{
return (__lhs < __rhs) ? __rhs : __lhs;
}
}
};
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA_FUNCTIONAL_MAXIMUM_H

77
qwen3_6_scripts/cccl_preload/include/cuda/__functional/minimum.h
View File

@@ -1,77 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_FUNCTIONAL_MINIMUM_H
#define _CUDA_FUNCTIONAL_MINIMUM_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__functional/minimum_maximum_common.h>
#include <cuda/std/__cmath/min_max.h>
#include <cuda/std/__type_traits/common_type.h>
#include <cuda/std/__type_traits/is_extended_floating_point.h>
#include <cuda/std/__type_traits/is_floating_point.h>
#include <cuda/std/__utility/ctad_support.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <class _Tp = void>
struct _CCCL_TYPE_VISIBILITY_DEFAULT minimum
{
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr _Tp operator()(const _Tp& __lhs, const _Tp& __rhs) const
noexcept(__is_maximum_minimum_noexcept_v<_Tp, _Tp, _Tp>)
{
if constexpr (::cuda::std::is_floating_point_v<_Tp> || ::cuda::std::__is_extended_floating_point_v<_Tp>)
{
return ::cuda::std::fmin(__lhs, __rhs);
}
else
{
return (__lhs < __rhs) ? __lhs : __rhs;
}
}
};
_CCCL_CTAD_SUPPORTED_FOR_TYPE(minimum);
template <>
struct _CCCL_TYPE_VISIBILITY_DEFAULT minimum<void>
{
_CCCL_EXEC_CHECK_DISABLE
template <class _Tp, class _Up, class _Common = ::cuda::std::common_type_t<_Tp, _Up>>
[[nodiscard]] _CCCL_API constexpr _Common operator()(const _Tp& __lhs, const _Up& __rhs) const
noexcept(__is_maximum_minimum_noexcept_v<_Tp, _Up, _Common>)
{
if constexpr (::cuda::std::is_floating_point_v<_Common> || ::cuda::std::__is_extended_floating_point_v<_Common>)
{
return ::cuda::std::fmin(static_cast<_Common>(__lhs), static_cast<_Common>(__rhs));
}
else
{
return (__lhs < __rhs) ? __lhs : __rhs;
}
}
};
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA_FUNCTIONAL_MINIMUM_H

52
qwen3_6_scripts/cccl_preload/include/cuda/__functional/minimum_maximum_common.h
View File

@@ -1,52 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_FUNCTIONAL_MINIMUM_MAXIMUM_COMMON_H
#define _CUDA_FUNCTIONAL_MINIMUM_MAXIMUM_COMMON_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__floating_point/traits.h>
#include <cuda/std/__type_traits/common_type.h>
#include <cuda/std/__type_traits/enable_if.h>
#include <cuda/std/__type_traits/is_nothrow_convertible.h>
#include <cuda/std/__utility/declval.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <typename _Tp, typename _Up, typename _Common = ::cuda::std::common_type_t<_Tp, _Up>, typename _Enable = void>
constexpr bool __is_maximum_minimum_noexcept_v =
noexcept(::cuda::std::declval<_Tp>() < ::cuda::std::declval<_Up>())
&& ::cuda::std::is_nothrow_convertible_v<_Tp, _Common> && ::cuda::std::is_nothrow_convertible_v<_Up, _Common>;
// Extended floating point types, such as __half and __nv bfloat16 cannot be compared with operator<. We need to
// handle them separately with SFINAE.
template <typename _Tp, typename _Up, typename _Common>
constexpr bool __is_maximum_minimum_noexcept_v<
_Tp,
_Up,
_Common,
::cuda::std::enable_if_t<::cuda::std::__is_ext_nv_fp_v<_Tp> || ::cuda::std::__is_ext_nv_fp_v<_Up>>> = false;
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA_FUNCTIONAL_MINIMUM_MAXIMUM_COMMON_H

652
qwen3_6_scripts/cccl_preload/include/cuda/__functional/operator_properties.h
View File

@@ -1,652 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_FUNCTIONAL_OPERATOR_PROPERTIES_H
#define _CUDA_FUNCTIONAL_OPERATOR_PROPERTIES_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__functional/maximum.h>
#include <cuda/__functional/minimum.h>
#include <cuda/__type_traits/is_floating_point.h>
#include <cuda/std/__floating_point/arithmetic.h>
#include <cuda/std/__floating_point/constants.h>
#include <cuda/std/__functional/operations.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/always_false.h>
#include <cuda/std/__type_traits/enable_if.h>
#include <cuda/std/__type_traits/is_extended_floating_point.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_same.h>
#include <cuda/std/__type_traits/remove_cv.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
/***********************************************************************************************************************
* Associativity
**********************************************************************************************************************/
template <class _Op>
[[nodiscard]] _CCCL_API constexpr bool __is_associative_static_assert()
{
static_assert(::cuda::std::__always_false_v<_Op>,
"operator_properties is not specialized for this operator and type combination");
return false;
}
template <class _Op, class _Tp, class Enable = void>
inline constexpr bool __is_associative_v = __is_associative_static_assert<_Op>();
// strictly speaking, plus (+) and multiply (*) are not associative because of overflow UB
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::std::plus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
true;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::std::plus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = false;
template <class _Tp>
inline constexpr bool __is_associative_v<::cuda::std::plus<>, _Tp> =
__is_associative_v<::cuda::std::plus<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool __is_associative_v<::cuda::std::multiplies<_Tp>,
_Tp,
::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> = true;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::std::multiplies<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> =
false;
template <class _Tp>
inline constexpr bool __is_associative_v<::cuda::std::multiplies<>, _Tp> =
__is_associative_v<::cuda::std::multiplies<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::std::bit_and<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
true;
template <class _Tp>
inline constexpr bool __is_associative_v<::cuda::std::bit_and<>, _Tp> =
__is_associative_v<::cuda::std::bit_and<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::std::bit_or<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
true;
template <class _Tp>
inline constexpr bool __is_associative_v<::cuda::std::bit_or<>, _Tp> =
__is_associative_v<::cuda::std::bit_or<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::std::bit_xor<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
true;
template <class _Tp>
inline constexpr bool __is_associative_v<::cuda::std::bit_xor<>, _Tp> =
__is_associative_v<::cuda::std::bit_xor<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::std::logical_and<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::is_same_v<_Tp, bool>>> =
true;
template <class _Tp>
inline constexpr bool __is_associative_v<::cuda::std::logical_and<>, _Tp> =
__is_associative_v<::cuda::std::logical_and<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::std::logical_or<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::is_same_v<_Tp, bool>>> =
true;
template <class _Tp>
inline constexpr bool __is_associative_v<::cuda::std::logical_or<>, _Tp> =
__is_associative_v<::cuda::std::logical_or<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::minimum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
true;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::minimum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = true;
template <class _Tp>
inline constexpr bool __is_associative_v<::cuda::minimum<>, _Tp> = __is_associative_v<::cuda::minimum<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::maximum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
true;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::maximum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = true;
template <class _Tp>
inline constexpr bool __is_associative_v<::cuda::maximum<>, _Tp> = __is_associative_v<::cuda::maximum<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::std::minus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
false;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::std::minus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = false;
template <class _Tp>
inline constexpr bool __is_associative_v<::cuda::std::minus<>, _Tp> =
__is_associative_v<::cuda::std::minus<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::std::divides<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
false;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::std::divides<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> =
false;
template <class _Tp>
inline constexpr bool __is_associative_v<::cuda::std::divides<>, _Tp> =
__is_associative_v<::cuda::std::divides<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_associative_v<::cuda::std::modulus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
false;
template <class _Tp>
inline constexpr bool __is_associative_v<::cuda::std::modulus<>, _Tp> =
__is_associative_v<::cuda::std::modulus<_Tp>, _Tp, void>;
template <class _Op, class _Tp>
inline constexpr bool is_associative_v = __is_associative_v<_Op, ::cuda::std::remove_cv_t<_Tp>>;
/***********************************************************************************************************************
* Commutativity
**********************************************************************************************************************/
template <class _Op>
[[nodiscard]] _CCCL_API constexpr bool __is_commutative_static_assert()
{
static_assert(::cuda::std::__always_false_v<_Op>,
"operator_properties is not specialized for this operator and type combination");
return false;
}
template <class _Op, class _Tp, class Enable = void>
inline constexpr bool __is_commutative_v = ::cuda::__is_commutative_static_assert<_Op>();
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::std::plus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
true;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::std::plus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = true;
template <class _Tp>
inline constexpr bool __is_commutative_v<::cuda::std::plus<>, _Tp> =
__is_commutative_v<::cuda::std::plus<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool __is_commutative_v<::cuda::std::multiplies<_Tp>,
_Tp,
::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> = true;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::std::multiplies<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> =
true;
template <class _Tp>
inline constexpr bool __is_commutative_v<::cuda::std::multiplies<>, _Tp> =
__is_commutative_v<::cuda::std::multiplies<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::std::bit_and<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
true;
template <class _Tp>
inline constexpr bool __is_commutative_v<::cuda::std::bit_and<>, _Tp> =
__is_commutative_v<::cuda::std::bit_and<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::std::bit_or<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
true;
template <class _Tp>
inline constexpr bool __is_commutative_v<::cuda::std::bit_or<>, _Tp> =
__is_commutative_v<::cuda::std::bit_or<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::std::bit_xor<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
true;
template <class _Tp>
inline constexpr bool __is_commutative_v<::cuda::std::bit_xor<>, _Tp> =
__is_commutative_v<::cuda::std::bit_xor<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::std::logical_and<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::is_same_v<_Tp, bool>>> =
true;
template <class _Tp>
inline constexpr bool __is_commutative_v<::cuda::std::logical_and<>, _Tp> =
__is_commutative_v<::cuda::std::logical_and<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::std::logical_or<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::is_same_v<_Tp, bool>>> =
true;
template <class _Tp>
inline constexpr bool __is_commutative_v<::cuda::std::logical_or<>, _Tp> =
__is_commutative_v<::cuda::std::logical_or<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::minimum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
true;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::minimum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = true;
template <class _Tp>
inline constexpr bool __is_commutative_v<::cuda::minimum<>, _Tp> = __is_commutative_v<::cuda::minimum<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::maximum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
true;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::maximum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = true;
template <class _Tp>
inline constexpr bool __is_commutative_v<::cuda::maximum<>, _Tp> = __is_commutative_v<::cuda::maximum<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::std::minus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
false;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::std::minus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = false;
template <class _Tp>
inline constexpr bool __is_commutative_v<::cuda::std::minus<>, _Tp> =
__is_commutative_v<::cuda::std::minus<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::std::divides<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
false;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::std::divides<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> =
false;
template <class _Tp>
inline constexpr bool __is_commutative_v<::cuda::std::divides<>, _Tp> =
__is_commutative_v<::cuda::std::divides<_Tp>, _Tp, void>;
template <class _Tp>
inline constexpr bool
__is_commutative_v<::cuda::std::modulus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
false;
template <class _Tp>
inline constexpr bool __is_commutative_v<::cuda::std::modulus<>, _Tp> =
__is_commutative_v<::cuda::std::modulus<_Tp>, _Tp, void>;
template <class _Op, class _Tp>
inline constexpr bool is_commutative_v = __is_commutative_v<_Op, ::cuda::std::remove_cv_t<_Tp>>;
/***********************************************************************************************************************
* Internal helpers
**********************************************************************************************************************/
template <typename>
inline constexpr bool __is_cuda_std_plus_v = false;
template <typename... _Tp>
inline constexpr bool __is_cuda_std_plus_v<::cuda::std::plus<_Tp...>> = true;
template <typename>
inline constexpr bool __is_cuda_std_multiplies_v = false;
template <typename... _Tp>
inline constexpr bool __is_cuda_std_multiplies_v<::cuda::std::multiplies<_Tp...>> = true;
template <typename>
inline constexpr bool __is_cuda_std_bit_and_v = false;
template <typename... _Tp>
inline constexpr bool __is_cuda_std_bit_and_v<::cuda::std::bit_and<_Tp...>> = true;
template <typename>
inline constexpr bool __is_cuda_std_bit_or_v = false;
template <typename... _Tp>
inline constexpr bool __is_cuda_std_bit_or_v<::cuda::std::bit_or<_Tp...>> = true;
template <typename>
inline constexpr bool __is_cuda_std_bit_xor_v = false;
template <typename... _Tp>
inline constexpr bool __is_cuda_std_bit_xor_v<::cuda::std::bit_xor<_Tp...>> = true;
template <typename>
inline constexpr bool __is_cuda_std_logical_and_v = false;
template <typename... _Tp>
inline constexpr bool __is_cuda_std_logical_and_v<::cuda::std::logical_and<_Tp...>> = true;
template <typename>
inline constexpr bool __is_cuda_std_logical_or_v = false;
template <typename... _Tp>
inline constexpr bool __is_cuda_std_logical_or_v<::cuda::std::logical_or<_Tp...>> = true;
template <typename>
inline constexpr bool __is_cuda_minimum_v = false;
template <typename... _Tp>
inline constexpr bool __is_cuda_minimum_v<::cuda::minimum<_Tp...>> = true;
template <typename>
inline constexpr bool __is_cuda_maximum_v = false;
template <typename... _Tp>
inline constexpr bool __is_cuda_maximum_v<::cuda::maximum<_Tp...>> = true;
/***********************************************************************************************************************
* Identity Element
**********************************************************************************************************************/
struct __no_identity_element
{
template <class _Tp>
[[nodiscard]] _CCCL_API constexpr bool operator==(_Tp&&) noexcept
{
return false;
}
template <class _Tp>
[[nodiscard]] _CCCL_API constexpr bool operator!=(_Tp&&) noexcept
{
return true;
}
};
template <class _Op, class _Tp>
[[nodiscard]] _CCCL_API constexpr auto identity_element() noexcept
{
using _Up = ::cuda::std::remove_cv_t<_Tp>;
if constexpr (__is_cuda_std_plus_v<_Op>)
{
if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
{
return _Up{};
}
else if constexpr (::cuda::is_floating_point_v<_Up>)
{
return ::cuda::std::__fp_neg(_Up{}); // -0.0 to preserve negative zero: -0.0 + (-0.0) = -0.0
}
else
{
return __no_identity_element{};
}
}
else if constexpr (__is_cuda_std_multiplies_v<_Op>)
{
if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_floating_point_v<_Up>
|| ::cuda::std::is_same_v<_Up, char>)
{
return _Up{1};
}
else if constexpr (::cuda::std::__is_extended_floating_point_v<_Up>)
{
return ::cuda::std::__fp_one<_Up>();
}
else
{
return __no_identity_element{};
}
}
else if constexpr (__is_cuda_std_bit_and_v<_Op>)
{
if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
{
return static_cast<_Up>(~_Up{});
}
else
{
return __no_identity_element{};
}
}
else if constexpr (__is_cuda_std_bit_or_v<_Op> || __is_cuda_std_bit_xor_v<_Op>)
{
if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
{
return _Up{};
}
else
{
return __no_identity_element{};
}
}
else if constexpr (__is_cuda_std_logical_and_v<_Op>)
{
if constexpr (::cuda::std::is_same_v<_Up, bool>)
{
return true;
}
else
{
return __no_identity_element{};
}
}
else if constexpr (__is_cuda_std_logical_or_v<_Op>)
{
if constexpr (::cuda::std::is_same_v<_Up, bool>)
{
return false;
}
else
{
return __no_identity_element{};
}
}
else if constexpr (__is_cuda_minimum_v<_Op>)
{
if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
{
return ::cuda::std::numeric_limits<_Up>::max();
}
else if constexpr (::cuda::is_floating_point_v<_Up>)
{
return ::cuda::std::numeric_limits<_Up>::infinity();
}
else
{
return __no_identity_element{};
}
}
else if constexpr (__is_cuda_maximum_v<_Op>)
{
if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
{
return ::cuda::std::numeric_limits<_Up>::lowest();
}
else if constexpr (::cuda::is_floating_point_v<_Up>)
{
return ::cuda::std::__fp_neg(::cuda::std::__fp_inf<_Up>());
}
else
{
return __no_identity_element{};
}
}
else
{
return __no_identity_element{};
}
}
template <class _Op, class _Tp, class = void>
inline constexpr bool has_identity_element_v = false;
template <class _Op, class _Tp>
inline constexpr bool has_identity_element_v<
_Op,
_Tp,
::cuda::std::enable_if_t<!::cuda::std::is_same_v<decltype(identity_element<_Op, _Tp>()), __no_identity_element>>> =
true;
/***********************************************************************************************************************
* Absorbing Element
**********************************************************************************************************************/
struct __no_absorbing_element
{};
template <class _Op, class _Tp>
[[nodiscard]] _CCCL_API constexpr auto absorbing_element() noexcept
{
using _Up = ::cuda::std::remove_cv_t<_Tp>;
if constexpr (__is_cuda_std_multiplies_v<_Op> || __is_cuda_std_bit_and_v<_Op>)
{
// Multiplication has no absorbing element for floating-point due to NaN, infinity,
// and -1.0 * +0.0 = -0.0 (!= +0.0).
if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
{
return _Up{};
}
else
{
return __no_absorbing_element{};
}
}
else if constexpr (__is_cuda_std_bit_or_v<_Op>)
{
if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
{
return static_cast<_Up>(~_Up{});
}
else
{
return __no_absorbing_element{};
}
}
else if constexpr (__is_cuda_std_logical_and_v<_Op>)
{
if constexpr (::cuda::std::is_same_v<_Up, bool>)
{
return false;
}
else
{
return __no_absorbing_element{};
}
}
else if constexpr (__is_cuda_std_logical_or_v<_Op>)
{
if constexpr (::cuda::std::is_same_v<_Up, bool>)
{
return true;
}
else
{
return __no_absorbing_element{};
}
}
else if constexpr (__is_cuda_minimum_v<_Op>)
{
if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
{
return ::cuda::std::numeric_limits<_Up>::lowest();
}
else if constexpr (::cuda::is_floating_point_v<_Up>)
{
return ::cuda::std::__fp_neg(::cuda::std::__fp_inf<_Up>());
}
else
{
return __no_absorbing_element{};
}
}
else if constexpr (__is_cuda_maximum_v<_Op>)
{
if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
{
return ::cuda::std::numeric_limits<_Up>::max();
}
else if constexpr (::cuda::is_floating_point_v<_Up>)
{
return ::cuda::std::__fp_inf<_Up>();
}
else
{
return __no_absorbing_element{};
}
}
else
{
return __no_absorbing_element{};
}
}
template <class _Op, class _Tp, class = void>
inline constexpr bool has_absorbing_element_v = false;
template <class _Op, class _Tp>
inline constexpr bool has_absorbing_element_v<
_Op,
_Tp,
::cuda::std::enable_if_t<!::cuda::std::is_same_v<decltype(absorbing_element<_Op, _Tp>()), __no_absorbing_element>>> =
true;
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA_FUNCTIONAL_OPERATOR_PROPERTIES_H

109
qwen3_6_scripts/cccl_preload/include/cuda/__functional/proclaim_return_type.h
View File

@@ -1,109 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___FUNCTIONAL_PROCLAIM_RETURN_TYPE_H
#define _CUDA___FUNCTIONAL_PROCLAIM_RETURN_TYPE_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__functional/invoke.h>
#include <cuda/std/__type_traits/decay.h>
#include <cuda/std/__type_traits/enable_if.h>
#include <cuda/std/__type_traits/is_same.h>
#include <cuda/std/__utility/forward.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
namespace __detail
{
template <class _Ret, class _DecayFn>
class __return_type_wrapper
{
private:
_DecayFn __fn_;
public:
__return_type_wrapper() = delete;
// NOLINTBEGIN(bugprone-forwarding-reference-overload)
_CCCL_TEMPLATE(class _Fn)
_CCCL_REQUIRES(::cuda::std::is_same_v<::cuda::std::decay_t<_Fn>, _DecayFn>)
_CCCL_API constexpr explicit __return_type_wrapper(_Fn&& __fn) noexcept
: __fn_(::cuda::std::forward<_Fn>(__fn))
{}
// NOLINTEND(bugprone-forwarding-reference-overload)
template <class... _As>
_CCCL_API constexpr _Ret operator()(_As&&... __as) & noexcept
{
#if !_CCCL_CUDA_COMPILER(NVCC) || defined(__CUDA_ARCH__)
static_assert(::cuda::std::is_same_v<_Ret, ::cuda::std::invoke_result_t<_DecayFn&, _As...>>,
"Return type shall match the proclaimed one exactly");
#endif // !_CCCL_CUDA_COMPILER(NVCC) || __CUDA_ARCH__
return ::cuda::std::__invoke(__fn_, ::cuda::std::forward<_As>(__as)...);
}
template <class... _As>
_CCCL_API constexpr _Ret operator()(_As&&... __as) && noexcept
{
#if !_CCCL_CUDA_COMPILER(NVCC) || defined(__CUDA_ARCH__)
static_assert(::cuda::std::is_same_v<_Ret, ::cuda::std::invoke_result_t<_DecayFn, _As...>>,
"Return type shall match the proclaimed one exactly");
#endif // !_CCCL_CUDA_COMPILER(NVCC) || __CUDA_ARCH__
return ::cuda::std::__invoke(::cuda::std::move(__fn_), ::cuda::std::forward<_As>(__as)...);
}
template <class... _As>
_CCCL_API constexpr _Ret operator()(_As&&... __as) const& noexcept
{
#if !_CCCL_CUDA_COMPILER(NVCC) || defined(__CUDA_ARCH__)
static_assert(::cuda::std::is_same_v<_Ret, ::cuda::std::invoke_result_t<const _DecayFn&, _As...>>,
"Return type shall match the proclaimed one exactly");
#endif // !_CCCL_CUDA_COMPILER(NVCC) || __CUDA_ARCH__
return ::cuda::std::__invoke(__fn_, ::cuda::std::forward<_As>(__as)...);
}
template <class... _As>
_CCCL_API constexpr _Ret operator()(_As&&... __as) const&& noexcept
{
#if !_CCCL_CUDA_COMPILER(NVCC) || defined(__CUDA_ARCH__)
static_assert(::cuda::std::is_same_v<_Ret, ::cuda::std::invoke_result_t<const _DecayFn, _As...>>,
"Return type shall match the proclaimed one exactly");
#endif // !_CCCL_CUDA_COMPILER(NVCC) || __CUDA_ARCH__
return ::cuda::std::__invoke(::cuda::std::move(__fn_), ::cuda::std::forward<_As>(__as)...);
}
};
} // namespace __detail
template <class _Ret, class _Fn>
_CCCL_API inline __detail::__return_type_wrapper<_Ret, ::cuda::std::decay_t<_Fn>>
proclaim_return_type(_Fn&& __fn) noexcept
{
return __detail::__return_type_wrapper<_Ret, ::cuda::std::decay_t<_Fn>>(::cuda::std::forward<_Fn>(__fn));
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___FUNCTIONAL_PROCLAIM_RETURN_TYPE_H

38
qwen3_6_scripts/cccl_preload/include/cuda/__fwd/get_memory_resource.h
View File

@@ -1,38 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___FWD_GET_MEMORY_RESOURCE_H
#define _CUDA___FWD_GET_MEMORY_RESOURCE_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_HAS_CTK()
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_MR
struct __get_memory_resource_t;
_CCCL_END_NAMESPACE_CUDA_MR
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_CTK()
#endif // _CUDA___FWD_GET_MEMORY_RESOURCE_H

38
qwen3_6_scripts/cccl_preload/include/cuda/__fwd/get_stream.h
View File

@@ -1,38 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___FWD_GET_STREAM_H
#define _CUDA___FWD_GET_STREAM_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_HAS_CTK()
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
struct get_stream_t;
_CCCL_END_NAMESPACE_CUDA
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_CTK()
#endif // _CUDA___FWD_GET_STREAM_H

141
qwen3_6_scripts/cccl_preload/include/cuda/__fwd/iterator.h
View File

@@ -1,141 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___FWD_ITERATOR_H
#define _CUDA___FWD_ITERATOR_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/random.h>
#include <cuda/std/__concepts/arithmetic.h>
#include <cuda/std/__concepts/copyable.h>
#include <cuda/std/__cstddef/types.h>
#include <cuda/std/__iterator/concepts.h>
#include <cuda/std/__iterator/incrementable_traits.h>
#include <cuda/std/__type_traits/enable_if.h>
#include <cuda/std/__type_traits/type_identity.h>
#include <cuda/std/cstdint>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <class _Tp, class _Index = ::cuda::std::ptrdiff_t>
class constant_iterator;
template <class _Tp>
[[nodiscard]] _CCCL_API _CCCL_CONSTEVAL auto __get_wider_signed() noexcept
{
if constexpr (sizeof(_Tp) < sizeof(int))
{
return ::cuda::std::type_identity<int>{};
}
else if constexpr (sizeof(_Tp) < sizeof(long))
{
return ::cuda::std::type_identity<long>{};
}
#if _CCCL_HAS_INT128()
else if constexpr (sizeof(_Tp) < sizeof(long long))
{
return ::cuda::std::type_identity<long long>{};
}
else // if constexpr (sizeof(_Start) < sizeof(__int128_t))
{
return ::cuda::std::type_identity<__int128_t>{};
}
#else // ^^^ _CCCL_HAS_INT128() ^^^ / vvv !_CCCL_HAS_INT128() vvv
else // if constexpr (sizeof(_Start) < sizeof(long long))
{
return ::cuda::std::type_identity<long long>{};
}
#endif // _CCCL_HAS_INT128()
}
template <class _Start>
using _IotaDiffT = typename ::cuda::std::conditional_t<
(!::cuda::std::integral<_Start> || sizeof(::cuda::std::iter_difference_t<_Start>) > sizeof(_Start)),
::cuda::std::type_identity<::cuda::std::iter_difference_t<_Start>>,
decltype(::cuda::__get_wider_signed<_Start>())>::type;
#if _CCCL_HAS_CONCEPTS()
template <::cuda::std::weakly_incrementable _Start, ::cuda::std::signed_integral _DiffT = _IotaDiffT<_Start>>
requires ::cuda::std::copyable<_Start>
#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv
template <class _Start,
class _DiffT = _IotaDiffT<_Start>,
::cuda::std::enable_if_t<::cuda::std::weakly_incrementable<_Start>, int> = 0,
::cuda::std::enable_if_t<::cuda::std::copyable<_Start>, int> = 0,
::cuda::std::enable_if_t<::cuda::std::signed_integral<_DiffT>, int> = 0>
#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^
class counting_iterator;
class discard_iterator;
template <class _Iter, class _Index = _Iter>
class permutation_iterator;
template <class _IndexType = ::cuda::std::size_t, class _Bijection = random_bijection<_IndexType>>
class shuffle_iterator;
template <class _Iter, class _Stride = ::cuda::std::iter_difference_t<_Iter>>
class strided_iterator;
template <class _Fn, class _Index = ::cuda::std::ptrdiff_t>
class tabulate_output_iterator;
template <class _InputFn, class _OutputFn, class _Iter>
class transform_input_output_iterator;
template <class _Fn, class _Iter>
class transform_iterator;
template <class _Fn, class _Iter>
class transform_output_iterator;
template <class... _Iterators>
class zip_iterator;
template <class>
inline constexpr bool __is_zip_iterator = false;
template <class... _Iterators>
inline constexpr bool __is_zip_iterator<zip_iterator<_Iterators...>> = true;
template <class _Fn>
class zip_function;
template <class>
inline constexpr bool __is_zip_function = false;
template <class _Fn>
inline constexpr bool __is_zip_function<zip_function<_Fn>> = true;
template <class _Fn, class... _Iterators>
class zip_transform_iterator;
template <class>
inline constexpr bool __is_zip_transform_iterator = false;
template <class _Fn, class... _Iterators>
inline constexpr bool __is_zip_transform_iterator<zip_transform_iterator<_Fn, _Iterators...>> = true;
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___FWD_ITERATOR_H

39
qwen3_6_scripts/cccl_preload/include/cuda/__fwd/random.h
View File

@@ -1,39 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___FWD_RANDOM_H
#define _CUDA___FWD_RANDOM_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/cstdint>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
class __feistel_bijection;
template <class _IndexType = ::cuda::std::uint64_t, class _Bijection = __feistel_bijection>
class random_bijection;
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___FWD_RANDOM_H

347
qwen3_6_scripts/cccl_preload/include/cuda/__iterator/constant_iterator.h
View File

@@ -1,347 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ITERATOR_CONSTANT_ITERATOR_H
#define _CUDA___ITERATOR_CONSTANT_ITERATOR_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/iterator.h>
#include <cuda/std/__concepts/constructible.h>
#include <cuda/std/__iterator/concepts.h>
#include <cuda/std/__iterator/iterator_traits.h>
#include <cuda/std/__ranges/compressed_movable_box.h>
#include <cuda/std/__type_traits/is_integral.h>
#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @addtogroup iterators
//! @{
//! @brief The @c constant_iterator class represents an iterator in an infinite sequence of repeated values.
//! @tparam _Tp the value type of the @c constant_iterator.
//! @tparam _Index The index type of the @c constant_iterator. It can optionally be specified, but must satisfy
//! __integer-like__
//!
//! This iterator is useful for creating a range filled with the same value without explicitly storing it in memory.
//! Using @c constant_iterator saves both memory capacity and bandwidth.
//!
//! The following code snippet demonstrates how to create a @c constant_iterator whose @c value_type is @c int and whose
//! value is @c 10.
//!
//! @code{.cpp}
//! #include <cuda/iterator>
//!
//! cuda::constant_iterator iter(10);
//!
//! *iter; // returns 10
//! iter[0]; // returns 10
//! iter[1]; // returns 10
//! iter[13]; // returns 10
//!
//! // and so on...
//! @endcode
template <class _Tp, class _Index>
class constant_iterator
{
private:
static_assert(::cuda::std::__integer_like<_Index>, "The index type of cuda::constant_iterator must be integer-like!");
// Not a base because then the friend operators would be ambiguous
::cuda::std::__compressed_movable_box<_Index, _Tp> __store_;
[[nodiscard]] _CCCL_API constexpr _Index& __index() noexcept
{
return __store_.template __get<0>();
}
[[nodiscard]] _CCCL_API constexpr const _Index& __index() const noexcept
{
return __store_.template __get<0>();
}
[[nodiscard]] _CCCL_API constexpr _Tp& __value() noexcept
{
return __store_.template __get<1>();
}
[[nodiscard]] _CCCL_API constexpr const _Tp& __value() const noexcept
{
return __store_.template __get<1>();
}
public:
using iterator_concept = ::cuda::std::random_access_iterator_tag;
using iterator_category = ::cuda::std::random_access_iterator_tag;
using value_type = _Tp;
using difference_type = ::cuda::std::ptrdiff_t;
// Those are technically not to spec, but pre-ranges iterator_traits do not work properly with iterators that do not
// define all 5 aliases, see https://en.cppreference.com/w/cpp/iterator/iterator_traits.html
using reference = _Tp;
using pointer = void;
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Tp2 = _Tp)
_CCCL_REQUIRES(::cuda::std::default_initializable<_Tp2>)
_CCCL_API constexpr constant_iterator() noexcept(::cuda::std::is_nothrow_default_constructible_v<_Tp2>)
: __store_()
{}
//! @brief Creates a @c constant_iterator from a value. The index is set to zero
//! @param __value The value to store in the @c constant_iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr constant_iterator(_Tp __value) noexcept(::cuda::std::is_nothrow_move_constructible_v<_Tp>)
: __store_(0, ::cuda::std::move(__value))
{}
//! @brief Creates @c constant_iterator from a value and an index
//! @param __value The value to store in the @c constant_iterator
//! @param __index The index in the sequence represented by this @c constant_iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(typename _Index2)
_CCCL_REQUIRES(::cuda::std::__integer_like<_Index2>)
_CCCL_API constexpr explicit constant_iterator(_Tp __value, _Index2 __index) noexcept(
::cuda::std::is_nothrow_move_constructible_v<_Tp>)
: __store_(static_cast<_Index>(__index), ::cuda::std::move(__value))
{}
//! @brief Returns a the current index
[[nodiscard]] _CCCL_API constexpr difference_type index() const noexcept
{
return static_cast<difference_type>(__index());
}
_CCCL_EXEC_CHECK_DISABLE
//! @brief Returns the stored value
[[nodiscard]] _CCCL_API constexpr reference operator*() const noexcept
{
return __value();
}
_CCCL_EXEC_CHECK_DISABLE
//! @brief Returns the stored value
[[nodiscard]] _CCCL_API constexpr reference operator[](difference_type) const noexcept
{
return __value();
}
//! @brief Increments the stored index
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr constant_iterator& operator++() noexcept
{
++__index();
return *this;
}
//! @brief Increments the stored index
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr constant_iterator operator++(int) noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Tp>)
{
auto __tmp = *this;
++__index();
return __tmp;
}
//! @brief Decrements the stored index
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr constant_iterator& operator--() noexcept
{
if constexpr (::cuda::std::is_signed_v<_Index> || !::cuda::std::is_integral_v<_Index>)
{
_CCCL_ASSERT(__index() > 0, "The index must be greater than or equal to 0");
}
--__index();
return *this;
}
//! @brief Decrements the stored index
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr constant_iterator operator--(int) noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Tp>)
{
if constexpr (::cuda::std::is_signed_v<_Index> || !::cuda::std::is_integral_v<_Index>)
{
_CCCL_ASSERT(__index() > 0, "The index must be greater than or equal to 0");
}
auto __tmp = *this;
--__index();
return __tmp;
}
//! @brief Advances a @c constant_iterator by a given number of elements
//! @param __n The amount of elements to advance
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr constant_iterator& operator+=(difference_type __n) noexcept
{
if constexpr (::cuda::std::is_signed_v<_Index> || !::cuda::std::is_integral_v<_Index>)
{
_CCCL_ASSERT(__index() + __n >= 0, "The index must be greater than or equal to 0");
}
__index() += static_cast<_Index>(__n);
return *this;
}
//! @brief Creates a copy of a @c constant_iterator advanced by a given number of elements
//! @param __iter The @c constant_iterator to advance
//! @param __n The amount of elements to advance
_CCCL_EXEC_CHECK_DISABLE
template <int = 0> // Must be template, or the compiler complains about a nonliteral return type
[[nodiscard]] _CCCL_API friend constexpr constant_iterator operator+(
const constant_iterator& __iter, difference_type __n) noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Tp>)
{
if constexpr (::cuda::std::is_signed_v<_Index>)
{
_CCCL_ASSERT(__iter.__index() + __n >= 0, "The index must be greater than or equal to 0");
}
return constant_iterator{__iter.__value(), __iter.__index() + __n};
}
//! @brief Creates a copy of a @c constant_iterator advanced by a given number of elements
//! @param __n The amount of elements to advance
//! @param __iter The @c constant_iterator to advance
_CCCL_EXEC_CHECK_DISABLE
template <int = 0> // Must be template, or the compiler complains about a nonliteral return type
[[nodiscard]] _CCCL_API friend constexpr constant_iterator operator+(
difference_type __n, const constant_iterator& __iter) noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Tp>)
{
if constexpr (::cuda::std::is_signed_v<_Index>)
{
_CCCL_ASSERT(__iter.__index() + __n >= 0, "The index must be greater than or equal to 0");
}
return constant_iterator{__iter.__value(), __iter.__index() + __n};
}
//! @brief Decrements a @c constant_iterator by a given number of elements
//! @param __n The amount of elements to decrement
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr constant_iterator& operator-=(difference_type __n) noexcept
{
if constexpr (::cuda::std::is_signed_v<_Index>)
{
_CCCL_ASSERT(__index() - __n >= 0, "The index must be greater than or equal to 0");
}
__index() -= static_cast<_Index>(__n);
return *this;
}
//! @brief Creates a copy of a @c constant_iterator decremented by a given number of elements
//! @param __n The amount of elements to decrement
//! @param __iter The @c constant_iterator to decrement
_CCCL_EXEC_CHECK_DISABLE
template <int = 0> // Must be template, or the compiler complains about a nonliteral return type
[[nodiscard]] _CCCL_API friend constexpr constant_iterator operator-(
const constant_iterator& __iter, difference_type __n) noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Tp>)
{
if constexpr (::cuda::std::is_signed_v<_Index>)
{
_CCCL_ASSERT(__iter.__index() - __n >= 0, "The index must be greater than or equal to 0");
}
return constant_iterator{__iter.__value(), __iter.__index() - __n};
}
//! @brief Returns the distance between two @c constant_iterator
[[nodiscard]] _CCCL_API friend constexpr difference_type
operator-(const constant_iterator& __lhs, const constant_iterator& __rhs) noexcept
{
return static_cast<difference_type>(__lhs.__index()) - static_cast<difference_type>(__rhs.__index());
}
//! @brief Compares two @c constant_iterator for equality by comparing the index in the sequence
[[nodiscard]] _CCCL_API friend constexpr bool
operator==(const constant_iterator& __lhs, const constant_iterator& __rhs) noexcept
{
return __lhs.__index() == __rhs.__index();
}
#if _CCCL_STD_VER <= 2017
//! @brief Compares two @c constant_iterator for inequality by comparing the index in the sequence
[[nodiscard]] _CCCL_API friend constexpr bool
operator!=(const constant_iterator& __lhs, const constant_iterator& __rhs) noexcept
{
return __lhs.__index() != __rhs.__index();
}
#endif // _CCCL_STD_VER <= 2017
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
//! @brief Three-way-compares two @c constant_iterator by comparing the index in the sequence
[[nodiscard]] _CCCL_API friend constexpr auto
operator<=>(const constant_iterator& __lhs, const constant_iterator& __rhs) noexcept
{
return __lhs.__index() <=> __rhs.__index();
}
#else // ^^^ _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() ^^^ / vvv !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR() vvv
//! @brief Compares two @c constant_iterator for less than by comparing the index in the sequence
[[nodiscard]] _CCCL_API friend constexpr bool
operator<(const constant_iterator& __lhs, const constant_iterator& __rhs) noexcept
{
return __lhs.__index() < __rhs.__index();
}
//! @brief Compares two @c constant_iterator for less equal by comparing the index in the sequence
[[nodiscard]] _CCCL_API friend constexpr bool
operator<=(const constant_iterator& __lhs, const constant_iterator& __rhs) noexcept
{
return __lhs.__index() <= __rhs.__index();
}
//! @brief Compares two @c constant_iterator for greater than by comparing the index in the sequence
[[nodiscard]] _CCCL_API friend constexpr bool
operator>(const constant_iterator& __lhs, const constant_iterator& __rhs) noexcept
{
return __lhs.__index() > __rhs.__index();
}
//! @brief Compares two @c constant_iterator for greater equal by comparing the index in the sequence
[[nodiscard]] _CCCL_API friend constexpr bool
operator>=(const constant_iterator& __lhs, const constant_iterator& __rhs) noexcept
{
return __lhs.__index() >= __rhs.__index();
}
#endif // !_LIBCUDACXX_HAS_NO_SPACESHIP_OPERATOR()
};
#ifndef _CCCL_DOXYGEN_INVOKED
template <class _Tp>
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES constant_iterator(_Tp) -> constant_iterator<_Tp, ::cuda::std::ptrdiff_t>;
_CCCL_TEMPLATE(class _Tp, typename _Index)
_CCCL_REQUIRES(::cuda::std::__integer_like<_Index>)
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES constant_iterator(_Tp, _Index) -> constant_iterator<_Tp, _Index>;
#endif // _CCCL_DOXYGEN_INVOKED
//! @brief Creates a @c constant_iterator from a value and an index
//! @param __value The value to be stored
//! @param __index The optional index representing the position in a sequence. Defaults to 0.
//! @relates constant_iterator
template <class _Tp, class _Index = ::cuda::std::ptrdiff_t>
[[nodiscard]] _CCCL_API constexpr auto make_constant_iterator(_Tp __value, _Index __index = 0)
{
return constant_iterator<_Tp, _Index>{::cuda::std::move(__value), __index};
}
//! @} // end iterators
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ITERATOR_CONSTANT_ITERATOR_H

552
qwen3_6_scripts/cccl_preload/include/cuda/__iterator/counting_iterator.h
View File

@@ -1,552 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ITERATOR_COUNTING_ITERATOR_H
#define _CUDA___ITERATOR_COUNTING_ITERATOR_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/iterator.h>
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
# include <cuda/std/__compare/three_way_comparable.h>
#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
#include <cuda/std/__concepts/arithmetic.h>
#include <cuda/std/__concepts/constructible.h>
#include <cuda/std/__concepts/convertible_to.h>
#include <cuda/std/__concepts/copyable.h>
#include <cuda/std/__concepts/equality_comparable.h>
#include <cuda/std/__concepts/invocable.h>
#include <cuda/std/__concepts/same_as.h>
#include <cuda/std/__concepts/semiregular.h>
#include <cuda/std/__concepts/totally_ordered.h>
#include <cuda/std/__functional/ranges_operations.h>
#include <cuda/std/__iterator/advance.h>
#include <cuda/std/__iterator/concepts.h>
#include <cuda/std/__iterator/distance.h>
#include <cuda/std/__iterator/incrementable_traits.h>
#include <cuda/std/__iterator/iterator_traits.h>
#include <cuda/std/__iterator/unreachable_sentinel.h>
#include <cuda/std/__ranges/enable_borrowed_range.h>
#include <cuda/std/__ranges/movable_box.h>
#include <cuda/std/__ranges/view_interface.h>
#include <cuda/std/__type_traits/always_false.h>
#include <cuda/std/__type_traits/conditional.h>
#include <cuda/std/__type_traits/enable_if.h>
#include <cuda/std/__type_traits/is_comparable.h>
#include <cuda/std/__type_traits/is_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
#include <cuda/std/__type_traits/make_unsigned.h>
#include <cuda/std/__type_traits/remove_cvref.h>
#include <cuda/std/__type_traits/type_identity.h>
#include <cuda/std/__type_traits/void_t.h>
#include <cuda/std/__utility/forward.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @addtogroup iterators
//! @{
//! @cond
template <class _Iter>
_CCCL_CONCEPT __decrementable = _CCCL_REQUIRES_EXPR((_Iter), _Iter __iter)(
requires(::cuda::std::incrementable<_Iter>), _Same_as(_Iter&)(--__iter), _Same_as(_Iter)(__iter--));
template <class _Iter>
_CCCL_CONCEPT __advanceable = _CCCL_REQUIRES_EXPR((_Iter), _Iter __iter, const _Iter __j, const _IotaDiffT<_Iter> __n)(
requires(__decrementable<_Iter>),
requires(::cuda::std::totally_ordered<_Iter>),
_Same_as(_Iter&) __iter += __n,
_Same_as(_Iter&) __iter -= __n,
requires(::cuda::std::is_constructible_v<_Iter, decltype(__j + __n)>),
requires(::cuda::std::is_constructible_v<_Iter, decltype(__n + __j)>),
requires(::cuda::std::is_constructible_v<_Iter, decltype(__j - __n)>),
requires(::cuda::std::convertible_to<decltype(__j - __j), _IotaDiffT<_Iter>>));
template <class, class = void>
struct __counting_iterator_category
{};
template <class _Tp>
struct __counting_iterator_category<_Tp, ::cuda::std::enable_if_t<::cuda::std::incrementable<_Tp>>>
{
using iterator_category = ::cuda::std::input_iterator_tag;
};
//! @endcond
//! @brief A @c counting_iterator represents an iterator into a range of sequentially increasing values.
//! @tparam _Start the value type of the @c counting_iterator.
//!
//! This iterator is useful for creating a range filled with a sequence without explicitly storing it in memory. Using
//! @c counting_iterator saves memory capacity and bandwidth.
//!
//! The following code snippet demonstrates how to create a @c counting_iterator whose @c value_type is @c int
//!
//! @code{.cpp}
//! #include <cuda/iterator>
//! ...
//! // create iterators
//! cuda::counting_iterator first(10);
//! cuda::counting_iterator last = first + 3;
//!
//! first[0] // returns 10
//! first[1] // returns 11
//! first[100] // returns 110
//!
//! // sum of [first, last)
//! std::reduce(first, last); // returns 33 (i.e. 10 + 11 + 12)
//!
//! // initialize vector to [0,1,2,..]
//! cuda::counting_iterator iter(0);
//! std::vector<int> vec(500);
//! std::copy(iter, iter + vec.size(), vec.begin());
//! @endcode
#if _CCCL_HAS_CONCEPTS()
template <::cuda::std::weakly_incrementable _Start, ::cuda::std::signed_integral _DiffT>
requires ::cuda::std::copyable<_Start>
#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv
template <class _Start,
class _DiffT,
::cuda::std::enable_if_t<::cuda::std::weakly_incrementable<_Start>, int>,
::cuda::std::enable_if_t<::cuda::std::copyable<_Start>, int>,
::cuda::std::enable_if_t<::cuda::std::signed_integral<_DiffT>, int>>
#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^
class counting_iterator : public __counting_iterator_category<_Start>
{
private:
_Start __value_ = _Start();
public:
using iterator_concept = ::cuda::std::conditional_t<
__advanceable<_Start>,
::cuda::std::random_access_iterator_tag,
::cuda::std::conditional_t<__decrementable<_Start>,
::cuda::std::bidirectional_iterator_tag,
::cuda::std::conditional_t<::cuda::std::incrementable<_Start>,
::cuda::std::forward_iterator_tag,
/*Else*/ ::cuda::std::input_iterator_tag>>>;
using value_type = _Start;
using difference_type = _DiffT;
// Those are technically not to spec, but pre-ranges iterator_traits do not work properly with iterators that do not
// define all 5 aliases, see https://en.cppreference.com/w/cpp/iterator/iterator_traits.html
using reference = _Start;
using pointer = void;
// Needed for comparison operators and constructors, because the other side might have a
// different difference type so we cannot reach into their private members. Usually you solve
// this with the power of friendship, but since this class uses concepts or SFINAE, spelling
// out the friendship is a faff.
//
// We also cannot use operator*() here to get the value because that imposes the additional
// burden of requiring _Start to be copy-constructible which is not needed for comparisons.
[[nodiscard]] _CCCL_API constexpr const _Start& __get_value() const noexcept
{
return __value_;
}
[[nodiscard]] _CCCL_API constexpr _Start& __get_value() noexcept
{
return __value_;
}
#if _CCCL_HAS_CONCEPTS()
_CCCL_HIDE_FROM_ABI counting_iterator()
requires ::cuda::std::default_initializable<_Start>
= default;
#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv
_CCCL_TEMPLATE(class _Start2 = _Start)
_CCCL_REQUIRES(::cuda::std::default_initializable<_Start2>)
_CCCL_API constexpr counting_iterator() noexcept(::cuda::std::is_nothrow_default_constructible_v<_Start2>) {}
#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^
//! @brief Creates a @c counting_iterator from an initial value.
//! @param __value The value to store in the @c counting_iterator
_CCCL_API constexpr explicit counting_iterator(_Start __value) noexcept(
::cuda::std::is_nothrow_move_constructible_v<_Start>)
: __value_(::cuda::std::move(__value))
{}
constexpr counting_iterator(const counting_iterator&) = default;
constexpr counting_iterator(counting_iterator&&) = default;
constexpr counting_iterator& operator=(const counting_iterator&) = default;
constexpr counting_iterator& operator=(counting_iterator&&) = default;
//! @brief Creates a @c counting_iterator from another @c counting_iterator of a different
//! difference type.
//! @param __other The @c counting_iterator to copy from.
_CCCL_TEMPLATE(class _DiffT2)
_CCCL_REQUIRES((!::cuda::std::same_as<_DiffT, _DiffT2>) )
_CCCL_API constexpr explicit counting_iterator(const counting_iterator<_Start, _DiffT2>& __other) noexcept(
::cuda::std::is_nothrow_copy_constructible_v<_Start>)
: __value_(__other.__get_value())
{}
//! @brief Creates a @c counting_iterator from another @c counting_iterator of a different
//! difference type.
//! @param __other The @c counting_iterator to move from.
_CCCL_TEMPLATE(class _DiffT2)
_CCCL_REQUIRES((!::cuda::std::same_as<_DiffT, _DiffT2>) )
_CCCL_API constexpr explicit counting_iterator(counting_iterator<_Start, _DiffT2>&& __other) noexcept(
::cuda::std::is_nothrow_move_constructible_v<_Start>)
: __value_(::cuda::std::move(__other.__get_value()))
{}
//! @brief Assignment between counting iterators of differing difference types is explicitly
//! deleted. If such a conversion is intended, use the copy or move constructors to convert.
_CCCL_TEMPLATE(class _DiffT2)
_CCCL_REQUIRES((!::cuda::std::same_as<_DiffT, _DiffT2>) )
_CCCL_API constexpr counting_iterator& operator=(const counting_iterator<_Start, _DiffT2>&) = delete;
//! @brief Assignment between counting iterators of differing difference types is explicitly
//! deleted. If such a conversion is intended, use the copy or move constructors
_CCCL_TEMPLATE(class _DiffT2)
_CCCL_REQUIRES((!::cuda::std::same_as<_DiffT, _DiffT2>) )
_CCCL_API constexpr counting_iterator& operator=(counting_iterator<_Start, _DiffT2>&&) = delete;
//! @brief Returns the value currently stored in the @c counting_iterator
[[nodiscard]] _CCCL_API constexpr _Start operator*() const
noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Start>)
{
return __value_;
}
//! @brief Returns the value currently stored in the @c counting_iterator advanced by a number of steps
//! @param __n The amount of elements to advance
_CCCL_TEMPLATE(class _Start2 = _Start)
_CCCL_REQUIRES(__advanceable<_Start2>)
[[nodiscard]] _CCCL_API constexpr _Start2 operator[](difference_type __n) const
noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Start2>
&& noexcept(::cuda::std::declval<const _Start2&>() + __n))
{
if constexpr (::cuda::std::__integer_like<_Start>)
{
return _Start(__value_ + static_cast<_Start>(__n));
}
else
{
return _Start(__value_ + __n);
}
}
//! @brief Increments the stored value
_CCCL_API constexpr counting_iterator& operator++() noexcept(noexcept(++::cuda::std::declval<_Start&>()))
{
++__value_;
return *this;
}
//! @brief Increments the stored value
_CCCL_API constexpr auto operator++(int) noexcept(
noexcept(++::cuda::std::declval<_Start&>()) && ::cuda::std::is_nothrow_copy_constructible_v<_Start>)
{
if constexpr (::cuda::std::incrementable<_Start>)
{
auto __tmp = *this;
++__value_;
return __tmp;
}
else
{
++__value_;
}
}
//! @brief Decrements the stored value
_CCCL_TEMPLATE(class _Start2 = _Start)
_CCCL_REQUIRES(__decrementable<_Start2>)
_CCCL_API constexpr counting_iterator& operator--() noexcept(noexcept(--::cuda::std::declval<_Start2&>()))
{
--__value_;
return *this;
}
//! @brief Decrements the stored value
_CCCL_TEMPLATE(class _Start2 = _Start)
_CCCL_REQUIRES(__decrementable<_Start2>)
_CCCL_API constexpr counting_iterator operator--(int) noexcept(
noexcept(--::cuda::std::declval<_Start2&>()) && ::cuda::std::is_nothrow_copy_constructible_v<_Start>)
{
auto __tmp = *this;
--*this;
return __tmp;
}
//! @brief Increments the stored value by a given number of elements
//! @param __n The number of elements to increment
_CCCL_API constexpr counting_iterator& operator+=(difference_type __n) noexcept(::cuda::std::__integer_like<_Start>)
{
if constexpr (::cuda::std::__integer_like<_Start> && !::cuda::std::__signed_integer_like<_Start>)
{
if (__n >= difference_type(0))
{
__value_ += static_cast<_Start>(__n);
}
else
{
__value_ -= static_cast<_Start>(-__n);
}
}
else if constexpr (::cuda::std::__signed_integer_like<_Start>)
{
__value_ += static_cast<_Start>(__n);
}
else
{
__value_ += __n;
}
return *this;
}
//! @brief Creates a copy of a @c counting_iterator advanced by a given number of elements
//! @param __iter The @c counting_iterator to advance
//! @param __n The amount of elements to advance
_CCCL_TEMPLATE(class _Start2 = _Start)
_CCCL_REQUIRES(__advanceable<_Start2>)
[[nodiscard]] _CCCL_API friend constexpr counting_iterator
operator+(counting_iterator __iter, difference_type __n) noexcept(::cuda::std::__integer_like<_Start2>)
{
__iter += __n;
return __iter;
}
//! @brief Creates a copy of a @c counting_iterator advanced by a given number of elements
//! @param __iter The @c counting_iterator to advance
//! @param __n The amount of elements to advance
_CCCL_TEMPLATE(class _Start2 = _Start)
_CCCL_REQUIRES(__advanceable<_Start2>)
[[nodiscard]] _CCCL_API friend constexpr counting_iterator
operator+(difference_type __n, counting_iterator __iter) noexcept(::cuda::std::__integer_like<_Start2>)
{
return __iter + __n;
}
//! @brief Decrements the stored value by a given number of elements
//! @param __n The amount of elements to decrement
_CCCL_TEMPLATE(class _Start2 = _Start)
_CCCL_REQUIRES(__advanceable<_Start2>)
_CCCL_API constexpr counting_iterator& operator-=(difference_type __n) noexcept(::cuda::std::__integer_like<_Start2>)
{
if constexpr (::cuda::std::__integer_like<_Start> && !::cuda::std::__signed_integer_like<_Start>)
{
if (__n >= difference_type(0))
{
__value_ -= static_cast<_Start>(__n);
}
else
{
__value_ += static_cast<_Start>(-__n);
}
}
else if constexpr (::cuda::std::__signed_integer_like<_Start>)
{
__value_ -= static_cast<_Start>(__n);
}
else
{
__value_ -= __n;
}
return *this;
}
//! @brief Creates a copy of a @c counting_iterator decremented by a given number of elements
//! @param __iter The @c counting_iterator to decrement
//! @param __n The amount of elements to decrement
_CCCL_TEMPLATE(class _Start2 = _Start)
_CCCL_REQUIRES(__advanceable<_Start2>)
[[nodiscard]] _CCCL_API friend constexpr counting_iterator
operator-(counting_iterator __iter, difference_type __n) noexcept(::cuda::std::__integer_like<_Start2>)
{
__iter -= __n;
return __iter;
}
//! @brief Returns the distance between two @c counting_iterator
//! @return The difference between the stored values
_CCCL_TEMPLATE(class _Start2 = _Start)
_CCCL_REQUIRES(__advanceable<_Start2>)
[[nodiscard]] _CCCL_API friend constexpr difference_type
operator-(const counting_iterator& __x, const counting_iterator& __y) noexcept(::cuda::std::__integer_like<_Start2>)
{
if constexpr (::cuda::std::__integer_like<_Start> && !::cuda::std::__signed_integer_like<_Start>)
{
if (__y.__value_ > __x.__value_)
{
return static_cast<difference_type>(-static_cast<difference_type>(__y.__value_ - __x.__value_));
}
return static_cast<difference_type>(__x.__value_ - __y.__value_);
}
else if constexpr (::cuda::std::__signed_integer_like<_Start>)
{
return static_cast<difference_type>(
static_cast<difference_type>(__x.__value_) - static_cast<difference_type>(__y.__value_));
}
else
{
return __x.__value_ - __y.__value_;
}
}
_CCCL_TEMPLATE(class _Start2, class _DiffT2)
_CCCL_REQUIRES(::cuda::std::equality_comparable_with<_Start, _Start2>)
[[nodiscard]] _CCCL_API friend constexpr bool
operator==(const counting_iterator& __x, const counting_iterator<_Start2, _DiffT2>& __y) noexcept(
noexcept(::cuda::std::__is_cpp17_nothrow_equality_comparable_v<_Start, _Start2>))
{
return __x.__value_ == __y.__get_value();
}
#if _CCCL_STD_VER <= 2017
//! @brief Compares two @c counting_iterator for inequality.
//! @return True if the stored values do not compare equal
_CCCL_TEMPLATE(class _Start2, class _DiffT2)
_CCCL_REQUIRES(::cuda::std::equality_comparable_with<_Start, _Start2>)
[[nodiscard]] _CCCL_API friend constexpr bool
operator!=(const counting_iterator& __x, const counting_iterator<_Start2, _DiffT2>& __y) noexcept(
noexcept(::cuda::std::declval<const _Start&>() != ::cuda::std::declval<const _Start2&>()))
{
return __x.__value_ != __y.__get_value();
}
#endif // _CCCL_STD_VER <= 2017
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
//! @brief Three-way compares two @c counting_iterator.
//! @return The three-way comparison of the stored values
template <class _Start2, class _DiffT2>
[[nodiscard]] _CCCL_API friend constexpr auto
operator<=>(const counting_iterator& __x, const counting_iterator<_Start2, _DiffT2>& __y) noexcept(
noexcept(::cuda::std::declval<const _Start&>() <=> ::cuda::std::declval<const _Start2&>()))
requires ::cuda::std::totally_ordered_with<_Start, _Start2>
&& ::cuda::std::three_way_comparable_with<_Start, _Start2>
{
return __x.__value_ <=> __y.__get_value();
}
#else // ^^^ _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() ^^^ / vvv !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR() vvv
//! @brief Compares two @c counting_iterator for less than.
//! @return True if stored values compare less than
_CCCL_TEMPLATE(class _Start2, class _DiffT2)
_CCCL_REQUIRES(::cuda::std::totally_ordered_with<_Start, _Start2>)
[[nodiscard]] _CCCL_API friend constexpr bool
operator<(const counting_iterator& __x, const counting_iterator<_Start2, _DiffT2>& __y) noexcept(
noexcept(::cuda::std::__is_cpp17_nothrow_less_than_comparable_v<_Start, _Start2>))
{
return __x.__value_ < __y.__get_value();
}
//! @brief Compares two @c counting_iterator for greater than.
//! @return True if stored values compare greater than
_CCCL_TEMPLATE(class _Start2, class _DiffT2)
_CCCL_REQUIRES(::cuda::std::totally_ordered_with<_Start, _Start2>)
[[nodiscard]] _CCCL_API friend constexpr bool
operator>(const counting_iterator& __x, const counting_iterator<_Start2, _DiffT2>& __y) noexcept(
::cuda::std::__is_cpp17_nothrow_less_than_comparable_v<_Start2, _Start>)
{
return __y < __x;
}
//! @brief Compares two @c counting_iterator for less equal.
//! @return True if stored values compare less equal
_CCCL_TEMPLATE(class _Start2, class _DiffT2)
_CCCL_REQUIRES(::cuda::std::totally_ordered_with<_Start, _Start2>)
[[nodiscard]] _CCCL_API friend constexpr bool
operator<=(const counting_iterator& __x, const counting_iterator<_Start2, _DiffT2>& __y) noexcept(
::cuda::std::__is_cpp17_nothrow_less_than_comparable_v<_Start2, _Start>)
{
return !(__y < __x);
}
//! @brief Compares two @c counting_iterator for greater equal.
//! @return True if stored values compare greater equal
_CCCL_TEMPLATE(class _Start2, class _DiffT2)
_CCCL_REQUIRES(::cuda::std::totally_ordered_with<_Start, _Start2>)
[[nodiscard]] _CCCL_API friend constexpr bool
operator>=(const counting_iterator& __x, const counting_iterator<_Start2, _DiffT2>& __y) noexcept(
::cuda::std::__is_cpp17_nothrow_less_than_comparable_v<_Start, _Start2>)
{
return !(__x < __y);
}
#endif // !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
};
//! @brief Creates a @c counting_iterator from an __integer-like__ @c _Start
//! @param __start The __integer-like__ @c _Start representing the initial count
//! @relates counting_iterator
template <class _Start>
[[nodiscard]] _CCCL_API constexpr auto make_counting_iterator(_Start __start)
{
return counting_iterator<_Start>{__start};
}
//! @} iterators
_CCCL_END_NAMESPACE_CUDA
#ifndef _CCCL_DOXYGEN_INVOKED
# if _CCCL_HAS_HOST_STD_LIB()
_CCCL_BEGIN_NAMESPACE_STD
//! counting_iterator is a C++20 iterator, so it does not play well with legacy STL features like std::distance
//! To work around that specialize those functions for counting_iterator
template <class _Diff, class _Start, class _DiffT2>
_CCCL_HOST_API constexpr void
advance(::cuda::counting_iterator<_Start, _DiffT2>& __iter, _Diff __diff) noexcept(::cuda::std::__integer_like<_Start>)
{
::cuda::std::advance(__iter, ::cuda::std::move(__diff));
}
template <class _Start, class _DiffT>
[[nodiscard]] _CCCL_HOST_API constexpr typename ::cuda::counting_iterator<_Start, _DiffT>::difference_type
distance(::cuda::counting_iterator<_Start, _DiffT> __first,
::cuda::counting_iterator<_Start, _DiffT> __last) noexcept(::cuda::std::__integer_like<_Start>)
{
return ::cuda::std::distance(::cuda::std::move(__first), ::cuda::std::move(__last));
}
template <class _Start, class _DiffT>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::counting_iterator<_Start, _DiffT>
next(::cuda::counting_iterator<_Start, _DiffT> __iter,
::cuda::std::type_identity_t<_DiffT> __n = 1) noexcept(::cuda::std::__integer_like<_Start>)
{
_CCCL_ASSERT(__n >= 0 || ::cuda::__decrementable<_Start>,
"Attempt to std::next(it, n) with negative n on a non-bidirectional iterator");
::cuda::std::advance(__iter, __n);
return __iter;
}
template <class _Start, class _DiffT>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::counting_iterator<_Start, _DiffT>
prev(::cuda::counting_iterator<_Start, _DiffT> __iter,
::cuda::std::type_identity_t<_DiffT> __n = 1) noexcept(::cuda::std::__integer_like<_Start>)
{
_CCCL_ASSERT(__n >= 0 || ::cuda::__decrementable<_Start>, "Attempt to std::prev(it, +n) on a non-bidi iterator");
::cuda::std::advance(__iter, -__n);
return __iter;
}
_CCCL_END_NAMESPACE_STD
# endif // _CCCL_HAS_HOST_STD_LIB()
#endif // _CCCL_DOXYGEN_INVOKED
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ITERATOR_COUNTING_ITERATOR_H

325
qwen3_6_scripts/cccl_preload/include/cuda/__iterator/discard_iterator.h
View File

@@ -1,325 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ITERATOR_DISCARD_ITERATOR_H
#define _CUDA___ITERATOR_DISCARD_ITERATOR_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/iterator.h>
#include <cuda/std/__iterator/concepts.h>
#include <cuda/std/__iterator/default_sentinel.h>
#include <cuda/std/__iterator/iterator_traits.h>
#include <cuda/std/__type_traits/is_same.h>
#include <cuda/std/__type_traits/remove_cvref.h>
#include <cuda/std/cstdint>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @addtogroup iterators
//! @{
//! @brief @c discard_iterator is an iterator which represents a special kind of pointer that ignores values written to
//! it upon dereference. This iterator is useful for ignoring the output of certain algorithms without wasting memory
//! capacity or bandwidth. @c discard_iterator may also be used to count the size of an algorithm's output which may not
//! be known a priori.
//!
//! The following code snippet demonstrates how to use @c discard_iterator to ignore one of the output ranges of
//! reduce_by_key
//!
//! @code
//! #include <cuda/iterator>
//! #include <thrust/reduce.h>
//! #include <thrust/device_vector.h>
//!
//! int main()
//! {
//! thrust::device_vector<int> keys{1, 3, 3, 3, 2, 2, 1};
//! thrust::device_vector<int> values{9, 8, 7, 6, 5, 4, 3};
//!
//! thrust::device_vector<int> result(4);
//!
//! // we are only interested in the reduced values
//! // use discard_iterator to ignore the output keys
//! thrust::reduce_by_key(keys.begin(), keys.end(),
//! values.begin(),
//! cuda::discard_iterator{},
//! result.begin());
//!
//! // result is now [9, 21, 9, 3]
//!
//! return 0;
//! }
//! @endcode
class discard_iterator
{
private:
::cuda::std::ptrdiff_t __index_ = 0;
public:
struct __discard_proxy
{
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES((!::cuda::std::is_same_v<::cuda::std::remove_cvref_t<_Tp>, __discard_proxy>) )
_CCCL_API constexpr const __discard_proxy& operator=(_Tp&&) const noexcept
{
return *this;
}
};
using iterator_concept = ::cuda::std::random_access_iterator_tag;
using iterator_category = ::cuda::std::random_access_iterator_tag;
using difference_type = ::cuda::std::ptrdiff_t;
using value_type = void;
using pointer = void;
using reference = void;
//! @brief Default constructs a @c discard_iterator at index zero
_CCCL_HIDE_FROM_ABI constexpr discard_iterator() = default;
//! @brief Constructs a @c discard_iterator with a given index
//! @param __index The index used for the discard iterator
_CCCL_TEMPLATE(class _Integer)
_CCCL_REQUIRES(::cuda::std::__integer_like<_Integer>)
_CCCL_API constexpr discard_iterator(_Integer __index) noexcept
: __index_(static_cast<::cuda::std::ptrdiff_t>(__index))
{}
//! @brief Returns the stored index
[[nodiscard]] _CCCL_API constexpr difference_type index() const noexcept
{
return __index_;
}
//! @brief Dereferences the @c discard_iterator returning a proxy that discards all values that are assigned to it
[[nodiscard]] _CCCL_API constexpr __discard_proxy operator*() const noexcept
{
return {};
}
//! @brief Subscipts the @c discard_iterator returning a proxy that discards all values that are assigned to it
[[nodiscard]] _CCCL_API constexpr __discard_proxy operator[](difference_type) const noexcept
{
return {};
}
//! @brief Increments the stored index
_CCCL_API constexpr discard_iterator& operator++() noexcept
{
++__index_;
return *this;
}
//! @brief Increments the stored index
_CCCL_API constexpr discard_iterator operator++(int) noexcept
{
discard_iterator __tmp = *this;
++__index_;
return __tmp;
}
//! @brief Decrements the stored index
_CCCL_API constexpr discard_iterator& operator--() noexcept
{
--__index_;
return *this;
}
//! @brief Decrements the stored index
_CCCL_API constexpr discard_iterator operator--(int) noexcept
{
discard_iterator __tmp = *this;
--__index_;
return __tmp;
}
//! @brief Returns a copy of this @c discard_iterator advanced by a number of elements
//! @param __n The number of elements to advance
[[nodiscard]] _CCCL_API constexpr discard_iterator operator+(difference_type __n) const noexcept
{
return discard_iterator{__index_ + __n};
}
//! @brief Returns a copy of a @c discard_iterator advanced by a number of elements
//! @param __n The number of elements to advance
//! @param __x The original @c discard_iterator
[[nodiscard]] _CCCL_API friend constexpr discard_iterator
operator+(difference_type __n, const discard_iterator& __x) noexcept
{
return __x + __n;
}
//! @brief Advances the index of this @c discard_iterator by a number of elements
//! @param __n The number of elements to advance
_CCCL_API constexpr discard_iterator& operator+=(difference_type __n) noexcept
{
__index_ += __n;
return *this;
}
//! @brief Returns a copy of this @c discard_iterator decremented by a number of elements
//! @param __n The number of elements to decrement
[[nodiscard]] _CCCL_API constexpr discard_iterator operator-(difference_type __n) const noexcept
{
return discard_iterator{__index_ - __n};
}
//! @brief Returns the distance between two @c discard_iterator's
//! @param __lhs The left @c discard_iterator
//! @param __rhs The right @c discard_iterator
//! @return __rhs.__index_ - __lhs.__index_
[[nodiscard]] _CCCL_API friend constexpr difference_type
operator-(const discard_iterator& __lhs, const discard_iterator& __rhs) noexcept
{
return __rhs.__index_ - __lhs.__index_;
}
//! @brief Returns the distance between a @c default_sentinel and a @c discard_iterator
//! @param __lhs The @c discard_iterator
//! @return -__lhs.__index_
[[nodiscard]] _CCCL_API friend constexpr difference_type
operator-(const discard_iterator& __lhs, ::cuda::std::default_sentinel_t) noexcept
{
return static_cast<difference_type>(-__lhs.__index_);
}
//! @brief Returns the distance between a @c discard_iterator and a @c default_sentinel
//! @param __rhs The @c discard_iterator
//! @return __rhs.__index_
[[nodiscard]] _CCCL_API friend constexpr difference_type
operator-(::cuda::std::default_sentinel_t, const discard_iterator& __rhs) noexcept
{
return static_cast<difference_type>(__rhs.__index_);
}
//! @brief Decrements the index of the @c discard_iterator by a number of elements
//! @param __n The number of elements to decrement
_CCCL_API constexpr discard_iterator& operator-=(difference_type __n) noexcept
{
__index_ -= __n;
return *this;
}
//! @brief Compares two @c discard_iterator for equality by comparing the indices
[[nodiscard]] _CCCL_API friend constexpr bool
operator==(const discard_iterator& __lhs, const discard_iterator& __rhs) noexcept
{
return __lhs.__index_ == __rhs.__index_;
}
#if _CCCL_STD_VER <= 2017
//! @brief Compares two @c discard_iterator for inequality by comparing the indices
[[nodiscard]] _CCCL_API friend constexpr bool
operator!=(const discard_iterator& __lhs, const discard_iterator& __rhs) noexcept
{
return __lhs.__index_ != __rhs.__index_;
}
#endif // _CCCL_STD_VER <= 2017
//! @brief Compares a @c discard_iterator with @c default_sentinel
//! @returns True if the index of @param __lhs is zero
[[nodiscard]] _CCCL_API friend constexpr bool
operator==(const discard_iterator& __lhs, ::cuda::std::default_sentinel_t) noexcept
{
return __lhs.__index_ == 0;
}
#if _CCCL_STD_VER <= 2017
//! @brief Compares a @c discard_iterator with @c default_sentinel
//! @returns True if the index of @param __lhs is zero
[[nodiscard]] _CCCL_API friend constexpr bool
operator==(::cuda::std::default_sentinel_t, const discard_iterator& __rhs) noexcept
{
return __rhs.__index_ == 0;
}
//! @brief Compares a @c discard_iterator with @c default_sentinel
//! @returns True if the index of @param __lhs is not zero
[[nodiscard]] _CCCL_API friend constexpr bool
operator!=(const discard_iterator& __lhs, ::cuda::std::default_sentinel_t) noexcept
{
return __lhs.__index_ != 0;
}
//! @brief Compares a @c discard_iterator with @c default_sentinel
//! @returns True if the index of @param __lhs is not zero
[[nodiscard]] _CCCL_API friend constexpr bool
operator!=(::cuda::std::default_sentinel_t, const discard_iterator& __rhs) noexcept
{
return __rhs.__index_ != 0;
}
#endif // _CCCL_STD_VER <= 2017
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
//! @brief Three-way-compares two @c discard_iterator by comparing the indices
[[nodiscard]] _CCCL_API friend constexpr strong_ordering
operator<=>(const discard_iterator& __lhs, const discard_iterator& __rhs) noexcept
{
return __lhs.__index_ <=> __rhs.__index_;
}
#else // ^^^ _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() ^^^ / vvv !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR() vvv
//! @brief Compares two @c discard_iterator for less than by comparing the indices
[[nodiscard]] _CCCL_API friend constexpr bool
operator<(const discard_iterator& __lhs, const discard_iterator& __rhs) noexcept
{
return __lhs.__index_ < __rhs.__index_;
}
//! @brief Compares two @c discard_iterator for less equal by comparing the indices
[[nodiscard]] _CCCL_API friend constexpr bool
operator<=(const discard_iterator& __lhs, const discard_iterator& __rhs) noexcept
{
return __lhs.__index_ <= __rhs.__index_;
}
//! @brief Compares two @c discard_iterator for greater than by comparing the indices
[[nodiscard]] _CCCL_API friend constexpr bool
operator>(const discard_iterator& __lhs, const discard_iterator& __rhs) noexcept
{
return __lhs.__index_ > __rhs.__index_;
}
//! @brief Compares two @c discard_iterator for greater equal by comparing the indices
[[nodiscard]] _CCCL_API friend constexpr bool
operator>=(const discard_iterator& __lhs, const discard_iterator& __rhs) noexcept
{
return __lhs.__index_ >= __rhs.__index_;
}
#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
};
//! @brief Creates a @c discard_iterator from an optional index.
//! @param __index The index of the @c discard_iterator within a range. The default index is @c 0.
//! @return A new @c discard_iterator with @c __index as the counter.
//! @relates discard_iterator
_CCCL_TEMPLATE(class _Integer = ::cuda::std::ptrdiff_t)
_CCCL_REQUIRES(::cuda::std::__integer_like<_Integer>)
[[nodiscard]] _CCCL_API constexpr discard_iterator make_discard_iterator(_Integer __index = 0)
{
return discard_iterator{__index};
}
//! @}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ITERATOR_DISCARD_ITERATOR_H

477
qwen3_6_scripts/cccl_preload/include/cuda/__iterator/permutation_iterator.h
View File

@@ -1,477 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ITERATOR_PERMUTATION_ITERATOR_H
#define _CUDA___ITERATOR_PERMUTATION_ITERATOR_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/iterator.h>
#include <cuda/std/__concepts/equality_comparable.h>
#include <cuda/std/__concepts/totally_ordered.h>
#include <cuda/std/__iterator/concepts.h>
#include <cuda/std/__iterator/iterator_traits.h>
#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
#include <cuda/std/__utility/declval.h>
#include <cuda/std/__utility/move.h>
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
# include <cuda/std/detail/libcxx/include/compare>
#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @addtogroup iterators
//! @{
//! @brief @c permutation_iterator is an iterator which represents a pointer into a reordered view of a given range.
//! @c permutation_iterator is an imprecise name; the reordered view need not be a strict permutation. This iterator is
//! useful for fusing a scatter or gather operation with other algorithms.
//!
//! This iterator takes two arguments:
//!
//! - an iterator to the range @c V on which the "permutation" will be applied, referred to as @c iter below
//! - an iterator to a range of indices defining the reindexing scheme that determines how the elements of @c V will
//! be permuted, referred to as @c index below
//!
//! Note that @c permutation_iterator is not limited to strict permutations of the given range @c V. The distance
//! between begin and end of the reindexing iterators is allowed to be smaller compared to the size of the range @c V,
//! in which case the @c permutation_iterator only provides a "permutation" of a subset of @c V. The indices do not
//! need to be unique. In this same context, it must be noted that the past-the-end @c permutation_iterator is
//! completely defined by means of the past-the-end iterator to the indices.
//!
//! The following code snippet demonstrates how to create a @c permutation_iterator which represents a reordering of the
//! contents of a @c device_vector.
//!
//! @code
//! #include <cuda/iterator>
//! #include <thrust/device_vector.h>
//! ...
//! thrust::device_vector<float> values{10.0f, 20.0f, 30.0f, 40.0f, 50.0f, 60.0f, 70.0f, 80.0f};
//! thrust::device_vector<int> indices{2, 6, 1, 3};
//!
//! using ElementIterator = thrust::device_vector<float>::iterator;
//! using IndexIterator = thrust::device_vector<int>::iterator;
//!
//! cuda::permutation_iterator<ElementIterator,IndexIterator> iter(values.begin(), indices.begin());
//!
//! *iter; // returns 30.0f;
//! iter[0]; // returns 30.0f;
//! iter[1]; // returns 70.0f;
//! iter[2]; // returns 20.0f;
//! iter[3]; // returns 40.0f;
//!
//! // iter[4] is an out-of-bounds error
//!
//! *iter = -1.0f; // sets values[2] to -1.0f;
//! iter[0] = -1.0f; // sets values[2] to -1.0f;
//! iter[1] = -1.0f; // sets values[6] to -1.0f;
//! iter[2] = -1.0f; // sets values[1] to -1.0f;
//! iter[3] = -1.0f; // sets values[3] to -1.0f;
//!
//! // values is now {10, -1, -1, -1, 50, 60, -1, 80}
//! @endcode
template <class _Iter, class _Index>
class permutation_iterator
{
private:
_Iter __iter_;
_Index __index_;
#ifndef _CCCL_DOXYGEN_INVOKED // Internal helpers
// We need to factor these out because old gcc chokes with using arguments in friend functions
template <class _Iter1>
static constexpr bool __nothrow_plus =
::cuda::std::is_nothrow_copy_constructible_v<_Iter1> && ::cuda::std::is_nothrow_copy_constructible_v<_Index>
&& noexcept(::cuda::std::declval<const _Index&>() + ::cuda::std::iter_difference_t<_Index>());
template <class _Iter1>
static constexpr bool __nothrow_minus =
::cuda::std::is_nothrow_copy_constructible_v<_Iter1> && ::cuda::std::is_nothrow_copy_constructible_v<_Index>
&& noexcept(::cuda::std::declval<const _Index&>() - ::cuda::std::iter_difference_t<_Index>());
template <class _Iter1>
static constexpr bool __nothrow_difference =
noexcept(::cuda::std::declval<_Iter1>() - ::cuda::std::declval<_Iter1>());
template <class _Iter1, class _Iter2>
static constexpr bool __nothrow_equality = noexcept(::cuda::std::declval<_Iter1>() == ::cuda::std::declval<_Iter2>());
template <class _Iter1, class _Iter2>
static constexpr bool __nothrow_less_than = noexcept(::cuda::std::declval<_Iter1>() < ::cuda::std::declval<_Iter2>());
template <class _Iter1, class _Iter2>
static constexpr bool __nothrow_less_equal =
noexcept(::cuda::std::declval<_Iter1>() <= ::cuda::std::declval<_Iter2>());
template <class _Iter1, class _Iter2>
static constexpr bool __nothrow_greater_than =
noexcept(::cuda::std::declval<_Iter1>() > ::cuda::std::declval<_Iter2>());
template <class _Iter1, class _Iter2>
static constexpr bool __nothrow_greater_equal =
noexcept(::cuda::std::declval<_Iter1>() >= ::cuda::std::declval<_Iter2>());
#endif // _CCCL_DOXYGEN_INVOKED
public:
using iterator_type = _Iter;
using iterator_concept = ::cuda::std::random_access_iterator_tag;
using iterator_category = ::cuda::std::random_access_iterator_tag;
using value_type = ::cuda::std::iter_value_t<_Iter>;
using __iter_difference_t = ::cuda::std::iter_difference_t<_Iter>;
using difference_type = ::cuda::std::iter_difference_t<_Index>;
using __index_value_t = ::cuda::std::iter_value_t<_Index>;
//! Ensure that the user passes an iterator to something interger_like
static_assert(::cuda::std::__integer_like<__index_value_t>,
"cuda::permutation_iterator: _Index must be an iterator to integer_like");
//! Ensure that the index value_type is convertible to difference_type
static_assert(::cuda::std::is_convertible_v<__index_value_t, difference_type>,
"cuda::permutation_iterator: _Indexs value type must be convertible to iter_difference<Iter>");
//! To actually use operator+ we need the index iterator to be random access
static_assert(::cuda::std::__has_random_access_traversal<_Index>,
"cuda::permutation_iterator: _Index must be a random access iterator!");
//! To actually use operator+ we need the base iterator to be random access
static_assert(::cuda::std::__has_random_access_traversal<_Iter>,
"cuda::permutation_iterator: _Iter must be a random access iterator!");
//! @brief Default constructs an @c permutation_iterator with a value initialized iterator and index
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter, class _Index2 = _Index)
_CCCL_REQUIRES(::cuda::std::default_initializable<_Iter2> _CCCL_AND ::cuda::std::default_initializable<_Index2>)
_CCCL_API constexpr permutation_iterator() noexcept(
::cuda::std::is_nothrow_default_constructible_v<_Iter2> && ::cuda::std::is_nothrow_default_constructible_v<_Index2>)
: __iter_()
, __index_()
{}
//! @brief Constructs an @c permutation_iterator from an iterator and an optional index
//! @param __iter The iterator to index from
//! @param __index The iterator with the permutations
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr permutation_iterator(_Iter __iter, _Index __index) noexcept(
::cuda::std::is_nothrow_copy_constructible_v<_Iter> && ::cuda::std::is_nothrow_copy_constructible_v<_Index>)
: __iter_(__iter)
, __index_(__index)
{}
//! @brief Returns a const reference to the stored base iterator @c iter
[[nodiscard]] _CCCL_API constexpr const _Iter& base() const& noexcept
{
return __iter_;
}
//! @brief Extracts the stored base iterator @c iter
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr _Iter base() && noexcept(::cuda::std::is_nothrow_move_constructible_v<_Iter>)
{
return ::cuda::std::move(__iter_);
}
//! @cond
//! @brief Returns a const reference to the stored index iterator @c index
[[nodiscard]] _CCCL_API constexpr const _Index& __index() const noexcept
{
return __index_;
}
//! @endcond
//! @brief Returns the current index
//! @return Equivalent to ``*index``
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr difference_type index() const noexcept
{
return static_cast<difference_type>(*__index_);
}
//! @brief Dereferences the @c permutation_iterator
//! @return Equivalent to ``iter[*index]``
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr decltype(auto)
operator*() noexcept(noexcept(__iter_[static_cast<__iter_difference_t>(*__index_)]))
{
return __iter_[static_cast<__iter_difference_t>(*__index_)];
}
//! @brief Dereferences the @c permutation_iterator
//! @return Equivalent to ``iter[*index]``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__dereferenceable<const _Iter2>)
[[nodiscard]] _CCCL_API constexpr decltype(auto) operator*() const
noexcept(noexcept(__iter_[static_cast<__iter_difference_t>(*__index_)]))
{
return __iter_[static_cast<__iter_difference_t>(*__index_)];
}
//! @brief Subscripts the @c permutation_iterator by an offset
//! @param __n The additional offset
//! @return Equivalent to ``iter[index[__n]]``
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr decltype(auto)
operator[](difference_type __n) noexcept(noexcept(__iter_[static_cast<__iter_difference_t>(__index_[__n])]))
{
return __iter_[static_cast<__iter_difference_t>(__index_[__n])];
}
//! @brief Subscripts the @c permutation_iterator by an offset
//! @param __n The additional offset
//! @return Equivalent to ``iter[index[__n]]``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__dereferenceable<const _Iter2>)
[[nodiscard]] _CCCL_API constexpr decltype(auto) operator[](difference_type __n) const
noexcept(noexcept(__iter_[static_cast<__iter_difference_t>(__index_[__n])]))
{
return __iter_[static_cast<__iter_difference_t>(__index_[__n])];
}
//! @brief Increments the @c permutation_iterator
//! @return Equivalent to ``++index``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr permutation_iterator& operator++() noexcept(noexcept(++__index_))
{
++__index_;
return *this;
}
//! @brief Increments the @c permutation_iterator
//! @return Equivalent to ``index++``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr permutation_iterator operator++(int) noexcept(
noexcept(++__index_)
&& ::cuda::std::is_nothrow_copy_constructible_v<_Iter> && ::cuda::std::is_nothrow_copy_constructible_v<_Index>)
{
permutation_iterator __tmp = *this;
++__index_;
return __tmp;
}
//! @brief Increments the @c permutation_iterator
//! @return Equivalent to ``--index``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr permutation_iterator& operator--() noexcept(noexcept(--__index_))
{
--__index_;
return *this;
}
//! @brief Increments the @c permutation_iterator
//! @return Equivalent to ``index++``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr permutation_iterator operator--(int) noexcept(
noexcept(--__index_)
&& ::cuda::std::is_nothrow_copy_constructible_v<_Iter> && ::cuda::std::is_nothrow_copy_constructible_v<_Index>)
{
permutation_iterator __tmp = *this;
--__index_;
return __tmp;
}
#ifndef _CCCL_DOXYGEN_INVOKED // Doxygen has issues with constexpr friend operators
//! @brief Advances a @c permutation_iterator by a given number of elements
//! @param __iter The original @c permutation_iterator
//! @param __n The number of elements to advance
//! @return Equivalent to ``permutation_iterator{iter, index + __n}``
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter> // Must be template, or the compiler complains about a nonliteral return type
[[nodiscard]] _CCCL_API friend constexpr permutation_iterator
operator+(const permutation_iterator& __iter, difference_type __n) noexcept(__nothrow_plus<_Iter2>)
{
return permutation_iterator{__iter.__iter_, __iter.__index_ + __n};
}
//! @brief Advances a @c permutation_iterator by a given number of elements
//! @param __n The number of elements to advance
//! @param __iter The original @c permutation_iterator
//! @return Equivalent to ``permutation_iterator{iter, index + __n}``
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter> // Must be template, or the compiler complains about a nonliteral return type
[[nodiscard]] _CCCL_API friend constexpr permutation_iterator
operator+(difference_type __n, const permutation_iterator& __iter) noexcept(__nothrow_plus<_Iter2>)
{
return permutation_iterator{__iter.__iter_, __iter.__index_ + __n};
}
//! @brief Decrements a @c permutation_iterator by a given number of elements
//! @param __iter The original @c permutation_iterator
//! @param __n The number of elements to decrement
//! @return Equivalent to ``permutation_iterator{iter, index - __n}``
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter> // Must be template, or the compiler complains about a nonliteral return type
[[nodiscard]] _CCCL_API friend constexpr permutation_iterator
operator-(const permutation_iterator& __iter, difference_type __n) noexcept(__nothrow_minus<_Iter2>)
{
return permutation_iterator{__iter.__iter_, __iter.__index_ - __n};
}
#endif // !_CCCL_DOXYGEN_INVOKED
//! @brief Advances the @c permutation_iterator by a given number of elements
//! @param __n The number of elements to advance
//! @return Equivalent to ``index + __n``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr permutation_iterator& operator+=(difference_type __n) noexcept(noexcept(__index_ += __n))
{
__index_ += __n;
return *this;
}
//! @brief Decrements the @c permutation_iterator by a given number of elements
//! @param __n The number of elements to decrement
//! @return Equivalent to ``index - __n``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr permutation_iterator& operator-=(difference_type __n) noexcept(noexcept(__index_ -= __n))
{
__index_ -= __n;
return *this;
}
//! @brief Returns the distance between two @c permutation_iterators.
//! @return Equivalent to ``__lhs.index - __rhs.index``
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API friend constexpr difference_type
operator-(const permutation_iterator& __lhs, const permutation_iterator& __rhs) noexcept(__nothrow_difference<_Index>)
{
return __lhs.__index_ - __rhs.__index();
}
//! @brief Compares two @c permutation_iterator for equality by comparing @c index
//! @return Equivalent to ``__lhs.index == __rhs.index``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _OtherIter, class _OtherOffset)
_CCCL_REQUIRES(::cuda::std::equality_comparable_with<_Iter, _OtherIter>)
[[nodiscard]] _CCCL_API friend constexpr bool operator==(
const permutation_iterator& __lhs,
const permutation_iterator<_OtherIter, _OtherOffset>& __rhs) noexcept(__nothrow_equality<_Index, _OtherOffset>)
{
return __lhs.__index_ == __rhs.__index();
}
#if _CCCL_STD_VER <= 2017
//! @brief Compares two @c permutation_iterator for inequality by comparing @c index
//! @return Equivalent to ``__lhs.index != __rhs.index``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _OtherIter, class _OtherOffset)
_CCCL_REQUIRES(::cuda::std::equality_comparable_with<_Iter, _OtherIter>)
[[nodiscard]] _CCCL_API friend constexpr bool operator!=(
const permutation_iterator& __lhs,
const permutation_iterator<_OtherIter, _OtherOffset>& __rhs) noexcept(__nothrow_equality<_Index, _OtherOffset>)
{
return !(__lhs.__index_ == __rhs.__index());
}
#endif // _CCCL_STD_VER <= 2017
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
template <class _Iter1, class _Iter2>
static constexpr bool __nothrow_three_way =
noexcept(::cuda::std::declval<_Iter1>() <=> ::cuda::std::declval<_Iter2>());
//! @brief Three-way-compares two @c permutation_iterator for inequality by comparing @c index
//! they point at
//! @return Equivalent to ``__lhs.index <=> __rhs.index``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _OtherIter, class _OtherOffset)
_CCCL_REQUIRES(::cuda::std::three_way_comparable_with<_Index, _OtherOffset>)
[[nodiscard]] _CCCL_API friend constexpr strong_ordering operator<=>(
const permutation_iterator& __lhs,
const permutation_iterator<_OtherIter, _OtherOffset>& __rhs) noexcept(__nothrow_three_way<_Index, _OtherOffset>)
{
return __lhs.__index_ <=> __rhs.__index();
}
#else // ^^^ _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() ^^^ / vvv !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR() vvv
//! @brief Compares two @c permutation_iterator for less than by comparing @c index
//! @return Equivalent to ``__lhs.index < __rhs.index``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _OtherIter, class _OtherOffset)
_CCCL_REQUIRES(::cuda::std::totally_ordered_with<_Index, _OtherOffset>)
[[nodiscard]] _CCCL_API friend constexpr bool operator<(
const permutation_iterator& __lhs,
const permutation_iterator<_OtherIter, _OtherOffset>& __rhs) noexcept(__nothrow_less_than<_Index, _OtherOffset>)
{
return __lhs.__index_ < __rhs.__index();
}
//! @brief Compares two @c permutation_iterator for less equal by comparing @c index
//! @return Equivalent to ``__lhs.index <= __rhs.index``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _OtherIter, class _OtherOffset)
_CCCL_REQUIRES(::cuda::std::totally_ordered_with<_Index, _OtherOffset>)
[[nodiscard]] _CCCL_API friend constexpr bool operator<=(
const permutation_iterator& __lhs,
const permutation_iterator<_OtherIter, _OtherOffset>& __rhs) noexcept(__nothrow_less_equal<_Index, _OtherOffset>)
{
return __lhs.__index_ <= __rhs.__index();
}
//! @brief Compares two @c permutation_iterator for greater than by comparing @c index
//! @return Equivalent to ``__lhs.index > __rhs.index``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _OtherIter, class _OtherOffset)
_CCCL_REQUIRES(::cuda::std::totally_ordered_with<_Index, _OtherOffset>)
[[nodiscard]] _CCCL_API friend constexpr bool operator>(
const permutation_iterator& __lhs,
const permutation_iterator<_OtherIter, _OtherOffset>& __rhs) noexcept(__nothrow_greater_than<_Index, _OtherOffset>)
{
return __lhs.__index_ > __rhs.__index();
}
//! @brief Compares two @c permutation_iterator for greater equal by comparing @c index
//! @return Equivalent to ``__lhs.index >= __rhs.index``
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _OtherIter, class _OtherOffset)
_CCCL_REQUIRES(::cuda::std::totally_ordered_with<_Index, _OtherOffset>)
[[nodiscard]] _CCCL_API friend constexpr bool operator>=(
const permutation_iterator& __lhs,
const permutation_iterator<_OtherIter, _OtherOffset>& __rhs) noexcept(__nothrow_greater_equal<_Index, _OtherOffset>)
{
return __lhs.__index_ >= __rhs.__index();
}
#endif // !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
};
#ifndef _CCCL_DOXYGEN_INVOKED
_CCCL_TEMPLATE(class _Iter, class _Index)
_CCCL_REQUIRES(
::cuda::std::__has_random_access_traversal<_Iter> _CCCL_AND ::cuda::std::__has_random_access_traversal<_Index>)
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES permutation_iterator(_Iter, _Index) -> permutation_iterator<_Iter, _Index>;
#endif // _CCCL_DOXYGEN_INVOKED
//! @brief Creates an @c permutation_iterator from a base iterator and an iterator to an integral index
//! @param __iter The iterator
//! @param __index The iterator to an integral index
//! @relates permutation_iterator
_CCCL_TEMPLATE(class _Iter, class _Index)
_CCCL_REQUIRES(
::cuda::std::__has_random_access_traversal<_Iter> _CCCL_AND ::cuda::std::__has_random_access_traversal<_Index>)
[[nodiscard]] _CCCL_API constexpr permutation_iterator<_Iter, _Index>
make_permutation_iterator(_Iter __iter, _Index __index) noexcept(
::cuda::std::is_nothrow_copy_constructible_v<_Iter> && ::cuda::std::is_nothrow_copy_constructible_v<_Index>)
{
return permutation_iterator<_Iter, _Index>{__iter, __index};
}
//! @}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ITERATOR_PERMUTATION_ITERATOR_H

363
qwen3_6_scripts/cccl_preload/include/cuda/__iterator/shuffle_iterator.h
View File

@@ -1,363 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ITERATOR_SHUFFLE_ITERATOR_H
#define _CUDA___ITERATOR_SHUFFLE_ITERATOR_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/iterator.h>
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
# include <cuda/std/__compare/three_way_comparable.h>
#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
#include <cuda/__random/random_bijection.h>
#include <cuda/std/__concepts/constructible.h>
#include <cuda/std/__random/is_valid.h>
#include <cuda/std/__type_traits/is_constructible.h>
#include <cuda/std/__type_traits/is_integral.h>
#include <cuda/std/__type_traits/is_nothrow_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
#include <cuda/std/__type_traits/is_same.h>
#include <cuda/std/__type_traits/make_signed.h>
#include <cuda/std/__utility/forward.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/cstdint>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
_CCCL_BEGIN_NV_DIAG_SUPPRESS(20011) // calling a __host__ function from a __host__ __device__ function shuffle_iterator
// is not allowed
//! @addtogroup iterators
//! @{
//! @brief Verifies that a given type @tparam _Bijection is a valid bijection function.
//! It verifies
//! * The bijection has a type alias ``index_type`` that satisfies ``integral``
//! * The bijection has a non-mutable member function size() that returns the number of elements as ``index_type``
//! * The bijection has a non-mutable call operator that takes a value of type ``index_type`` in the range
//! ``[0, size())`` and projects it into the range ``[0, size())``
template <class _Bijection>
_CCCL_CONCEPT __is_bijection = _CCCL_REQUIRES_EXPR((_Bijection), const _Bijection& __fun)(
typename(typename _Bijection::index_type),
requires(::cuda::std::is_integral_v<typename _Bijection::index_type>),
requires(::cuda::std::is_same_v<decltype(__fun.size()), typename _Bijection::index_type>),
requires(
::cuda::std::is_same_v<decltype(__fun(typename _Bijection::index_type(0))), typename _Bijection::index_type>));
//! @brief @c shuffle_iterator is an iterator which generates a sequence of integral values representing a random
//! permutation.
//! @tparam _IndexType The type of the index to shuffle. Defaults to uint64_t
//! @tparam _BijectionFunc The bijection to use. This should be a bijective function that maps [0..n) -> [0..n). It must
//! be deterministic and stateless. Defaults to cuda::random_biijection<_IndexType>
//!
//! @c shuffle_iterator is an iterator which generates a sequence of values representing a random permutation. This
//! iterator is useful for working with random permutations of a range without explicitly storing them in memory. The
//! shuffle iterator is also useful for sampling from a range by selecting only a subset of the elements in the
//! permutation.
//!
//! The following code snippet demonstrates how to create a @c shuffle_iterator which generates a random permutation
//! of the range[0, 4)
//!
//! @code
//! #include <cuda/iterator>
//! ...
//! // create a shuffle_iterator
//! cuda::shuffle_iterator iterator{cuda::random_bijection{4, cuda::std::minstd_rand(0xDEADBEEF)}};
//! // iterator[0] returns 1
//! // iterator[1] returns 3
//! // iterator[2] returns 2
//! // iterator[3] returns 0
//! @endcode
template <class _IndexType, class _Bijection>
class shuffle_iterator
{
private:
_Bijection __bijection_;
_IndexType __current_;
static_assert(::cuda::std::is_integral_v<_IndexType>, "_IndexType must be an integral type");
static_assert(__is_bijection<_Bijection>, "_Bijection must be a valid bijection function");
public:
using iterator_category = ::cuda::std::random_access_iterator_tag;
using iterator_concept = ::cuda::std::random_access_iterator_tag;
using value_type = _IndexType;
using difference_type = ::cuda::std::make_signed_t<value_type>;
// Those are technically not to spec, but pre-ranges iterator_traits do not work properly with iterators that do not
// define all 5 aliases, see https://en.cppreference.com/w/cpp/iterator/iterator_traits.html
using reference = _IndexType;
using pointer = void;
_CCCL_EXEC_CHECK_DISABLE // NVCC 12.0 fails to default construct when _CCCL_TEMPLATE is used
template <class _Bijection2 = _Bijection,
::cuda::std::enable_if_t<::cuda::std::default_initializable<_Bijection2>, int> = 0>
_CCCL_API constexpr shuffle_iterator() noexcept(::cuda::std::is_nothrow_default_constructible_v<_Bijection2>)
: __bijection_()
, __current_(0)
{}
//! @brief Constructs a @c shuffle_iterator from a given bijection and an optional start position
//! @param __bijection The bijection representing the shuffled integer sequence
//! @param __start The position of the iterator in the shuffled integer sequence
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr shuffle_iterator(_Bijection __bijection, value_type __start = 0) noexcept(
::cuda::std::is_nothrow_move_constructible_v<_Bijection>)
: __bijection_(::cuda::std::move(__bijection))
, __current_(__start)
{}
//! @brief Constructs a @c shuffle_iterator by constructing the bijection function in place and an optional start
//! position
//! @param __num_elements The size of the bijection sequence
//! @param __gen The random number generator to initialize the bijection
//! @param __start The optional stating index of the @c shuffle_iterator in the bijection sequence
_CCCL_EXEC_CHECK_DISABLE
template <class _RNG> // constraining here breaks CTAD
_CCCL_API explicit constexpr shuffle_iterator(value_type __num_elements, _RNG&& __gen, value_type __start = 0) //
noexcept(::cuda::std::is_nothrow_constructible_v<_Bijection, value_type, _RNG>)
: __bijection_(__num_elements, ::cuda::std::forward<_RNG>(__gen))
, __current_(__start)
{}
//! @brief Dereferences the @c shuffle_iterator by invoking the bijection with the stored index
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr value_type operator*() const noexcept(noexcept(__bijection_(0)))
{
_CCCL_ASSERT(__current_ < static_cast<value_type>(__bijection_.size()),
"shuffle_iterator::operator*: Trying to dereference a shuffle_iterator past the end!");
return static_cast<value_type>(__bijection_(static_cast<typename _Bijection::index_type>(__current_)));
}
//! @brief Subscripts the @c shuffle_iterator by invoking the bijection with the stored index advanced by a given
//! number of elements
//! @param __n The additional number of elements
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr value_type operator[](difference_type __n) const noexcept(noexcept(__bijection_(0)))
{
_CCCL_ASSERT(static_cast<value_type>(static_cast<difference_type>(__current_) + __n)
< static_cast<value_type>(__bijection_.size()),
"shuffle_iterator::operator*: Trying to subscript a shuffle_iterator past the end!");
return static_cast<value_type>(__bijection_(static_cast<typename _Bijection::index_type>(__current_ + __n)));
}
//! @brief Increments the @c shuffle_iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr shuffle_iterator& operator++() noexcept
{
++__current_;
return *this;
}
//! @brief Increments the @c shuffle_iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr shuffle_iterator operator++(int) noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Bijection>)
{
auto __tmp = *this;
++__current_;
return __tmp;
}
//! @brief Decrements the @c shuffle_iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr shuffle_iterator& operator--() noexcept
{
--__current_;
return *this;
}
//! @brief Decrements the @c shuffle_iterator
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr shuffle_iterator
operator--(int) noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Bijection>)
{
auto __tmp = *this;
--__current_;
return __tmp;
}
//! @brief Advances the @c shuffle_iterator by a given number of elements
//! @param __n The number of elements to advance
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr shuffle_iterator& operator+=(difference_type __n) noexcept
{
#if _CCCL_COMPILER(MSVC) // C4308: negative integral constant converted to unsigned type
__current_ = static_cast<value_type>(static_cast<difference_type>(__current_) + __n);
#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
__current_ += __n;
#endif // !_CCCL_COMPILER(MSVC)
return *this;
}
//! @brief Returns a copy of a @c shuffle_iterator incremented by a given number of elements
//! @param __iter The @c shuffle_iterator to copy
//! @param __n The number of elements to increment
_CCCL_EXEC_CHECK_DISABLE
template <int = 0> // Must be template, or the compiler complains about a nonliteral return type
[[nodiscard]]
_CCCL_API friend constexpr shuffle_iterator operator+(shuffle_iterator __iter, difference_type __n) noexcept
{
#if _CCCL_COMPILER(MSVC) // C4308: negative integral constant converted to unsigned type
__iter.__current_ = static_cast<value_type>(static_cast<difference_type>(__iter.__current_) + __n);
#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
__iter.__current_ += __n;
#endif // !_CCCL_COMPILER(MSVC)
return __iter;
}
//! @brief Returns a copy of a @c shuffle_iterator incremented by a given number of elements
//! @param __n The number of elements to increment
//! @param __iter The @c shuffle_iterator to copy
_CCCL_EXEC_CHECK_DISABLE
template <int = 0> // Must be template, or the compiler complains about a nonliteral return type
[[nodiscard]]
_CCCL_API friend constexpr shuffle_iterator operator+(difference_type __n, shuffle_iterator __iter) noexcept
{
#if _CCCL_COMPILER(MSVC) // C4308: negative integral constant converted to unsigned type
__iter.__current_ = static_cast<value_type>(static_cast<difference_type>(__iter.__current_) + __n);
#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
__iter.__current_ += __n;
#endif // !_CCCL_COMPILER(MSVC)
return __iter;
}
//! @brief Decrements the @c shuffle_iterator by a given number of elements
//! @param __n The number of elements to decrement
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr shuffle_iterator& operator-=(difference_type __n) noexcept
{
#if _CCCL_COMPILER(MSVC) // C4308: negative integral constant converted to unsigned type
__current_ = static_cast<value_type>(static_cast<difference_type>(__current_) - __n);
#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
__current_ -= __n;
#endif // !_CCCL_COMPILER(MSVC)
return *this;
}
//! @brief Returns a copy of a @c shuffle_iterator decremented by a given number of elements
//! @param __iter The @c shuffle_iterator to copy
//! @param __n The number of elements to decrement
_CCCL_EXEC_CHECK_DISABLE
template <int = 0> // Must be template, or the compiler complains about a nonliteral return type
[[nodiscard]]
_CCCL_API friend constexpr shuffle_iterator operator-(shuffle_iterator __iter, difference_type __n) noexcept
{
#if _CCCL_COMPILER(MSVC) // C4308: negative integral constant converted to unsigned type
__iter.__current_ = static_cast<value_type>(static_cast<difference_type>(__iter.__current_) - __n);
#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
__iter.__current_ -= __n;
#endif // !_CCCL_COMPILER(MSVC)
return __iter;
}
//! @brief Calculates the distance between two @c shuffle_iterator
[[nodiscard]] _CCCL_API friend constexpr difference_type
operator-(const shuffle_iterator& __x, const shuffle_iterator& __y) noexcept
{
return static_cast<difference_type>(__x.__current_ - __y.__current_);
}
//! @brief Compares two @c shuffle_iterator for equality by comparing their index
[[nodiscard]] _CCCL_API friend constexpr bool
operator==(const shuffle_iterator& __x, const shuffle_iterator& __y) noexcept
{
return __x.__current_ == __y.__current_;
}
#if _CCCL_STD_VER <= 2017
//! @brief Compares two @c shuffle_iterator for inequality by comparing their index
[[nodiscard]] _CCCL_API friend constexpr bool
operator!=(const shuffle_iterator& __x, const shuffle_iterator& __y) noexcept
{
return __x.__current_ != __y.__current_;
}
#endif // _CCCL_STD_VER <= 2017
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
//! @brief Three-way compares two @c shuffle_iterator for less equal by comparing their index
[[nodiscard]] _CCCL_API friend constexpr ::cuda::std::strong_ordering
operator<=>(const shuffle_iterator& __x, const shuffle_iterator& __y) noexcept
{
return __x.__current_ <=> __y.__current_;
}
#else // ^^^ _LIBCUDACXX_HAS_NO_SPACESHIP_OPERATOR ^^^ / vvv !_LIBCUDACXX_HAS_NO_SPACESHIP_OPERATOR vvv
//! @brief Compares two @c shuffle_iterator for less than by comparing their index
[[nodiscard]] _CCCL_API friend constexpr bool
operator<(const shuffle_iterator& __x, const shuffle_iterator& __y) noexcept
{
return __x.__current_ < __y.__current_;
}
//! @brief Compares two @c shuffle_iterator for greater than by comparing their index
[[nodiscard]] _CCCL_API friend constexpr bool
operator>(const shuffle_iterator& __x, const shuffle_iterator& __y) noexcept
{
return __x.__current_ > __y.__current_;
}
//! @brief Compares two @c shuffle_iterator for less equal by comparing their index
[[nodiscard]] _CCCL_API friend constexpr bool
operator<=(const shuffle_iterator& __x, const shuffle_iterator& __y) noexcept
{
return __x.__current_ <= __y.__current_;
}
//! @brief Compares two @c shuffle_iterator for greater equal by comparing their index
[[nodiscard]] _CCCL_API friend constexpr bool
operator>=(const shuffle_iterator& __x, const shuffle_iterator& __y) noexcept
{
return __x.__current_ >= __y.__current_;
}
#endif // !_LIBCUDACXX_HAS_NO_SPACESHIP_OPERATOR
};
_CCCL_TEMPLATE(class _Bijection)
_CCCL_REQUIRES(__is_bijection<_Bijection>)
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES shuffle_iterator(_Bijection)
-> shuffle_iterator<typename _Bijection::index_type, _Bijection>;
_CCCL_TEMPLATE(class _Bijection, typename _Integral)
_CCCL_REQUIRES(__is_bijection<_Bijection> _CCCL_AND ::cuda::std::is_integral_v<_Integral>)
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES shuffle_iterator(_Bijection, _Integral)
-> shuffle_iterator<typename _Bijection::index_type, _Bijection>;
//! @brief make_shuffle_iterator creates a @c shuffle_iterator from an integer and a bijection function
//! @param __fun The bijection function used for shuffling
//! @param __start The starting position of the @c shuffle_iterator
//! @relates shuffle_iterator
template <class _Bijection, class _IndexType>
[[nodiscard]] _CCCL_API constexpr auto make_shuffle_iterator(_Bijection __fun, _IndexType __start = 0)
{
return shuffle_iterator<_IndexType, _Bijection>{::cuda::std::move(__fun), __start};
}
//! @}
_CCCL_END_NV_DIAG_SUPPRESS()
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ITERATOR_TRANSFORM_ITERATOR_H

435
qwen3_6_scripts/cccl_preload/include/cuda/__iterator/strided_iterator.h
View File

@@ -1,435 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ITERATOR_STRIDED_ITERATOR_H
#define _CUDA___ITERATOR_STRIDED_ITERATOR_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/iterator.h>
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
# include <cuda/std/__compare/three_way_comparable.h>
#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
#include <cuda/std/__concepts/constructible.h>
#include <cuda/std/__concepts/equality_comparable.h>
#include <cuda/std/__concepts/totally_ordered.h>
#include <cuda/std/__iterator/iterator_traits.h>
#include <cuda/std/__mdspan/submdspan_helper.h>
#include <cuda/std/__ranges/compressed_movable_box.h>
#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
#include <cuda/std/__utility/declval.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @addtogroup iterators
//! @{
//! @brief A @c strided_iterator wraps another iterator and advances it by a specified stride each time it is
//! incremented or decremented.
//!
//! @tparam _Iter A random access iterator
//! @tparam _Stride Either an <a href="https://eel.is/c++draft/iterator.concept.winc#4">integer-like</a> or an
//! <a href="https://eel.is/c++draft/views.contiguous#concept:integral-constant-like">integral-constant-like</a>
//! specifying the stride
template <class _Iter, class _Stride>
class strided_iterator
{
private:
static_assert(::cuda::std::__has_random_access_traversal<_Iter>,
"The iterator underlying a strided_iterator must be a random access iterator.");
static_assert(::cuda::std::__integer_like<_Stride> || ::cuda::std::__integral_constant_like<_Stride>,
"The stride of a strided_iterator must either be an integer-like or integral-constant-like.");
template <class, class>
friend class strided_iterator;
// Not a base because then the friend operators would be ambiguous
::cuda::std::__compressed_movable_box<_Iter, _Stride> __store_;
[[nodiscard]] _CCCL_API constexpr _Iter& __iter() noexcept
{
return __store_.template __get<0>();
}
[[nodiscard]] _CCCL_API constexpr const _Iter& __iter() const noexcept
{
return __store_.template __get<0>();
}
[[nodiscard]] _CCCL_API constexpr _Stride& __stride() noexcept
{
return __store_.template __get<1>();
}
[[nodiscard]] _CCCL_API constexpr const _Stride& __stride() const noexcept
{
return __store_.template __get<1>();
}
public:
using iterator_concept = ::cuda::std::random_access_iterator_tag;
using iterator_category = ::cuda::std::random_access_iterator_tag;
using value_type = ::cuda::std::iter_value_t<_Iter>;
using difference_type = ::cuda::std::iter_difference_t<_Iter>;
// Those are technically not to spec, but pre-ranges iterator_traits do not work properly with iterators that do not
// define all 5 aliases, see https://en.cppreference.com/w/cpp/iterator/iterator_traits.html
using reference = ::cuda::std::iter_reference_t<_Iter>;
using pointer = void;
//! @brief value-initializes both the base iterator and stride
//! @note _Iter must be default initializable because it is a random_access_iterator and thereby semiregular
//! _Stride must be integer-like or integral_constant_like which requires default constructability
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter, class _Stride2 = _Stride)
_CCCL_REQUIRES(::cuda::std::default_initializable<_Iter2> _CCCL_AND ::cuda::std::default_initializable<_Stride2>)
_CCCL_API constexpr strided_iterator() noexcept(::cuda::std::is_nothrow_default_constructible_v<_Iter2>
&& ::cuda::std::is_nothrow_default_constructible_v<_Stride2>)
: __store_()
{}
//! @brief Constructs a @c strided_iterator from a base iterator
//! @param __iter The base iterator
//! @note We cannot construct a @c strided_iterator with an
//! <a href="https://eel.is/c++draft/iterator.concept.winc#4">integer-like</a> stride, because that would value
//! construct to 0 and incrementing the iterator would do nothing.
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Stride2 = _Stride)
_CCCL_REQUIRES(::cuda::std::__integral_constant_like<_Stride2>)
_CCCL_API constexpr explicit strided_iterator(_Iter __iter) noexcept(
::cuda::std::is_nothrow_move_constructible_v<_Iter> && ::cuda::std::is_nothrow_default_constructible_v<_Stride2>)
: __store_(::cuda::std::move(__iter))
{}
//! @brief Constructs a @c strided_iterator from a base iterator and a stride
//! @param __iter The base iterator
//! @param __stride The new stride
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr explicit strided_iterator(_Iter __iter, _Stride __stride) noexcept(
::cuda::std::is_nothrow_move_constructible_v<_Iter> && ::cuda::std::is_nothrow_move_constructible_v<_Stride>)
: __store_(::cuda::std::move(__iter), ::cuda::std::move(__stride))
{}
//! @brief Returns a const reference to the stored iterator
[[nodiscard]] _CCCL_API constexpr const _Iter& base() const& noexcept
{
return __iter();
}
//! @brief Extracts the stored iterator
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr _Iter base() && noexcept(::cuda::std::is_nothrow_move_constructible_v<_Iter>)
{
return ::cuda::std::move(__iter());
}
static constexpr bool __noexcept_stride =
noexcept(static_cast<difference_type>(::cuda::std::__de_ice(::cuda::std::declval<const _Stride&>())));
//! @brief Returns the current stride as an integral value
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr difference_type stride() const noexcept(__noexcept_stride)
{
return static_cast<difference_type>(::cuda::std::__de_ice(__stride()));
}
//! @brief Dereferences the stored base iterator
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr decltype(auto) operator*() noexcept(noexcept(*::cuda::std::declval<_Iter&>()))
{
return *__iter();
}
//! @brief Dereferences the stored base iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__dereferenceable<const _Iter2>)
[[nodiscard]] _CCCL_API constexpr decltype(auto) operator*() const
noexcept(noexcept(*::cuda::std::declval<const _Iter2&>()))
{
return *__iter();
}
//! @brief Subscripts the stored base iterator with a given offset times the stride
//! @param __n The offset
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr decltype(auto)
operator[](difference_type __n) noexcept(__noexcept_stride && noexcept(::cuda::std::declval<_Iter&>()[__n]))
{
return __iter()[__n * stride()];
}
//! @brief Subscripts the stored base iterator with a given offset times the stride
//! @param __n The offset
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__dereferenceable<const _Iter2>)
[[nodiscard]] _CCCL_API constexpr decltype(auto) operator[](difference_type __n) const
noexcept(__noexcept_stride && noexcept(::cuda::std::declval<const _Iter2&>()[__n]))
{
return __iter()[__n * stride()];
}
//! @brief Increments the stored base iterator by the stride
// Note: we cannot use __iter() += stride() in the noexcept clause because that breaks gcc < 9
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr strided_iterator&
operator++() noexcept(__noexcept_stride && noexcept(::cuda::std::declval<_Iter&>() += 1))
{
__iter() += stride();
return *this;
}
//! @brief Increments the stored base iterator by the stride
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr auto operator++(int) noexcept(
noexcept(__noexcept_stride && noexcept(::cuda::std::declval<_Iter&>() += 1))
&& ::cuda::std::is_nothrow_copy_constructible_v<_Iter> && ::cuda::std::is_nothrow_copy_constructible_v<_Stride>)
{
auto __tmp = *this;
__iter() += stride();
return __tmp;
}
//! @brief Decrements the stored base iterator by the stride
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr strided_iterator&
operator--() noexcept(__noexcept_stride && noexcept(::cuda::std::declval<_Iter&>() -= 1))
{
__iter() -= stride();
return *this;
}
//! @brief Decrements the stored base iterator by the stride
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr strided_iterator operator--(int) noexcept(
noexcept(__noexcept_stride && noexcept(::cuda::std::declval<_Iter&>() -= 1))
&& ::cuda::std::is_nothrow_copy_constructible_v<_Iter> && ::cuda::std::is_nothrow_copy_constructible_v<_Stride>)
{
auto __tmp = *this;
__iter() -= stride();
return __tmp;
}
//! @brief Advances a @c strided_iterator by a given number of steps
//! @param __n The number of steps to increment
//! @note Increments the base iterator by @c __n times the stride
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr strided_iterator&
operator+=(difference_type __n) noexcept(__noexcept_stride && noexcept(::cuda::std::declval<_Iter&>() += 1))
{
__iter() += stride() * __n;
return *this;
}
template <class _Iter2>
static constexpr bool __nothrow_plus =
::cuda::std::is_nothrow_copy_constructible_v<_Iter2>
&& noexcept(::cuda::std::declval<const _Iter2&>() + difference_type());
//! @brief Returns a copy of a @c strided_iterator incremented by a given number of steps
//! @param __iter The @c strided_iterator to advance
//! @param __n The number of steps to increment
_CCCL_EXEC_CHECK_DISABLE
template <int = 0> // Must be template, or the compiler complains about a nonliteral return type
[[nodiscard]]
_CCCL_API friend constexpr strided_iterator
operator+(const strided_iterator& __iter, difference_type __n) noexcept(__nothrow_plus<_Iter>)
{
return strided_iterator{__iter.__iter() + __iter.stride() * __n, __iter.__stride()};
}
//! @brief Returns a copy of a @c strided_iterator incremented by a given number of steps
//! @param __n The number of steps to increment
//! @param __iter The @c strided_iterator to advance
_CCCL_EXEC_CHECK_DISABLE
template <int = 0> // Must be template, or the compiler complains about a nonliteral return type
[[nodiscard]]
_CCCL_API friend constexpr strided_iterator
operator+(difference_type __n, const strided_iterator& __iter) noexcept(__nothrow_plus<_Iter>)
{
return strided_iterator{__iter.__iter() + __iter.stride() * __n, __iter.__stride()};
}
//! @brief Decrements a @c strided_iterator by a given number of steps
//! @param __n The number of steps to decrement
//! @note Decrements the base iterator by @c __n times the stride
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr strided_iterator&
operator-=(difference_type __n) noexcept(__noexcept_stride && noexcept(::cuda::std::declval<_Iter&>() -= 1))
{
__iter() -= stride() * __n;
return *this;
}
template <class _Iter2>
static constexpr bool __nothrow_minus =
::cuda::std::is_nothrow_copy_constructible_v<_Iter2>
&& noexcept(::cuda::std::declval<const _Iter2&>() - difference_type());
//! @brief Returns a copy of a @c strided_iterator decremented by a given number of steps
//! @param __n The number of steps to decrement
//! @param __iter The @c strided_iterator to decrement
_CCCL_EXEC_CHECK_DISABLE
template <int = 0> // Must be template, or the compiler complains about a nonliteral return type
[[nodiscard]]
_CCCL_API friend constexpr strided_iterator
operator-(const strided_iterator& __iter, difference_type __n) noexcept(__nothrow_minus<_Iter>)
{
return strided_iterator{__iter.__iter() - __iter.stride() * __n, __iter.__stride()};
}
template <class _Iter2, class _OtherIter>
static constexpr bool __noexcept_difference =
noexcept(::cuda::std::declval<const _Iter2&>() - ::cuda::std::declval<const _OtherIter&>());
//! @brief Returns distance between two @c strided_iterator's in units of the stride
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _OtherIter, class _OtherStride)
_CCCL_REQUIRES(::cuda::std::sized_sentinel_for<_OtherIter, _Iter>)
[[nodiscard]] _CCCL_API friend constexpr difference_type
operator-(const strided_iterator& __x, const strided_iterator<_OtherIter, _OtherStride>& __y) //
noexcept(__noexcept_difference<_Iter, _OtherIter>)
{
const difference_type __diff = __x.__iter() - __y.base();
_CCCL_ASSERT(__x.stride() == __y.stride(), "Taking the difference of two strided_iterators with different stride");
_CCCL_ASSERT(__diff % __x.stride() == 0, "Underlying iterator difference must be divisible by the stride");
return __diff / __x.stride();
}
//! @brief Compares two @c strided_iterator's for equality by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _OtherIter, class _OtherStride)
_CCCL_REQUIRES(::cuda::std::equality_comparable_with<_Iter, _OtherIter>)
[[nodiscard]] _CCCL_API friend constexpr bool
operator==(const strided_iterator& __x, const strided_iterator<_OtherIter, _OtherStride>& __y) noexcept(
noexcept(::cuda::std::declval<const _Iter&>() == ::cuda::std::declval<const _OtherIter&>()))
{
return __x.__iter() == __y.base();
}
#if _CCCL_STD_VER <= 2017
//! @brief Compares two @c strided_iterator's for inequality by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _OtherIter, class _OtherStride)
_CCCL_REQUIRES(::cuda::std::equality_comparable_with<_Iter, _OtherIter>)
[[nodiscard]] _CCCL_API friend constexpr bool
operator!=(const strided_iterator& __x, const strided_iterator<_OtherIter, _OtherStride>& __y) noexcept(
noexcept(::cuda::std::declval<const _Iter&>() == ::cuda::std::declval<const _OtherIter&>()))
{
return __x.__iter() != __y.base();
}
#endif // _CCCL_STD_VER <= 2017
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
//! @brief Threeway-compares two @c strided_iterator's by comparing the stored iterators
_CCCL_TEMPLATE(class _OtherIter, class _OtherStride)
_CCCL_REQUIRES(::cuda::std::totally_ordered_with<_Iter, _OtherIter>)
_CCCL_REQUIRES(
::cuda::std::totally_ordered<_Iter, _OtherIter> _CCCL_AND ::cuda::std::three_way_comparable_with<_Iter, _OtherIter>)
[[nodiscard]] _CCCL_API friend constexpr auto
operator<=>(const strided_iterator& __x, const strided_iterator<_OtherIter, _OtherStride>& __y) noexcept(
noexcept(::cuda::std::declval<const _Iter&>() <=> ::cuda::std::declval<const _OtherIter&>()))
{
return __x.__iter() <=> __y.base();
}
#else // ^^^ _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() ^^^ / vvv !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR() vvv
//! @brief Compares two @c strided_iterator's for less than by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _OtherIter, class _OtherStride)
_CCCL_REQUIRES(::cuda::std::totally_ordered_with<_Iter, _OtherIter>)
[[nodiscard]] _CCCL_API friend constexpr bool
operator<(const strided_iterator& __x, const strided_iterator<_OtherIter, _OtherStride>& __y) noexcept(
noexcept(::cuda::std::declval<const _Iter&>() < ::cuda::std::declval<const _OtherIter&>()))
{
return __x.__iter() < __y.base();
}
//! @brief Compares two @c strided_iterator's for greater than by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _OtherIter, class _OtherStride)
_CCCL_REQUIRES(::cuda::std::totally_ordered_with<_Iter, _OtherIter>)
[[nodiscard]] _CCCL_API friend constexpr bool
operator>(const strided_iterator& __x, const strided_iterator<_OtherIter, _OtherStride>& __y) noexcept(
noexcept(::cuda::std::declval<const _Iter&>() < ::cuda::std::declval<const _OtherIter&>()))
{
return __y < __x;
}
//! @brief Compares two @c strided_iterator's for less equal by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _OtherIter, class _OtherStride)
_CCCL_REQUIRES(::cuda::std::totally_ordered_with<_Iter, _OtherIter>)
[[nodiscard]] _CCCL_API friend constexpr bool
operator<=(const strided_iterator& __x, const strided_iterator<_OtherIter, _OtherStride>& __y) noexcept(
noexcept(::cuda::std::declval<const _Iter&>() < ::cuda::std::declval<const _OtherIter&>()))
{
return !(__y < __x);
}
//! @brief Compares two @c strided_iterator's for greater equal by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _OtherIter, class _OtherStride)
_CCCL_REQUIRES(::cuda::std::totally_ordered_with<_Iter, _OtherIter>)
[[nodiscard]] _CCCL_API friend constexpr bool
operator>=(const strided_iterator& __x, const strided_iterator<_OtherIter, _OtherStride>& __y) noexcept(
noexcept(::cuda::std::declval<const _Iter&>() < ::cuda::std::declval<const _OtherIter&>()))
{
return !(__x < __y);
}
#endif // !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
};
#ifndef _CCCL_DOXYGEN_INVOKED
template <class _Iter, typename _Stride>
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES strided_iterator(_Iter, _Stride) -> strided_iterator<_Iter, _Stride>;
#endif // _CCCL_DOXYGEN_INVOKED
//! @brief Creates a @c strided_iterator from a random access iterator
//! @param __iter The random_access iterator
//! @relates strided_iterator
_CCCL_TEMPLATE(class _Stride, class _Iter)
_CCCL_REQUIRES(::cuda::std::__integral_constant_like<_Stride>)
[[nodiscard]] _CCCL_API constexpr auto make_strided_iterator(_Iter __iter)
{
return strided_iterator<_Iter, _Stride>{::cuda::std::move(__iter)};
}
//! @brief Creates a @c strided_iterator from a random access iterator and a stride
//! @param __iter The random_access iterator
//! @param __stride The new stride
//! @relates strided_iterator
template <class _Iter, class _Stride>
[[nodiscard]] _CCCL_API constexpr auto make_strided_iterator(_Iter __iter, _Stride __stride)
{
return strided_iterator<_Iter, _Stride>{::cuda::std::move(__iter), __stride};
}
//! @} // end iterators
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ITERATOR_STRIDED_ITERATOR_H

381
qwen3_6_scripts/cccl_preload/include/cuda/__iterator/tabulate_output_iterator.h
View File

@@ -1,381 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ITERATOR_TABULATE_OUTPUT_ITERATOR_H
#define _CUDA___ITERATOR_TABULATE_OUTPUT_ITERATOR_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/iterator.h>
#include <cuda/std/__concepts/constructible.h>
#include <cuda/std/__functional/invoke.h>
#include <cuda/std/__iterator/concepts.h>
#include <cuda/std/__iterator/iterator_traits.h>
#include <cuda/std/__ranges/compressed_movable_box.h>
#include <cuda/std/__ranges/movable_box.h>
#include <cuda/std/__type_traits/conditional.h>
#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
#include <cuda/std/__type_traits/is_same.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__type_traits/remove_cvref.h>
#include <cuda/std/__utility/forward.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/cstdint>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @addtogroup iterators
//! @{
template <class _Fn, class _Index>
class __tabulate_proxy
{
private:
template <class, class>
friend class tabulate_output_iterator;
_Fn& __func_;
_Index __index_;
_CCCL_API constexpr explicit __tabulate_proxy(_Fn& __func, _Index __index) noexcept
: __func_(__func)
, __index_(__index)
{}
public:
_CCCL_HIDE_FROM_ABI __tabulate_proxy(const __tabulate_proxy&) = default;
_CCCL_HIDE_FROM_ABI __tabulate_proxy& operator=(const __tabulate_proxy&) = default;
_CCCL_HIDE_FROM_ABI __tabulate_proxy(__tabulate_proxy&&) = default;
_CCCL_HIDE_FROM_ABI __tabulate_proxy& operator=(__tabulate_proxy&&) = default;
_CCCL_TEMPLATE(class _Arg)
_CCCL_REQUIRES(::cuda::std::is_invocable_v<_Fn&, _Index, _Arg>)
_CCCL_API constexpr const __tabulate_proxy&
operator=(_Arg&& __arg) noexcept(::cuda::std::is_nothrow_invocable_v<_Fn&, _Index, _Arg>)
{
::cuda::std::invoke(__func_, __index_, ::cuda::std::forward<_Arg>(__arg));
return *this;
}
_CCCL_TEMPLATE(class _Arg)
_CCCL_REQUIRES(::cuda::std::is_invocable_v<const _Fn&, _Index, _Arg>)
_CCCL_API constexpr const __tabulate_proxy& operator=(_Arg&& __arg) const
noexcept(::cuda::std::is_nothrow_invocable_v<const _Fn&, _Index, _Arg>)
{
::cuda::std::invoke(__func_, __index_, ::cuda::std::forward<_Arg>(__arg));
return *this;
}
};
//! @brief @c tabulate_output_iterator is a special kind of output iterator which, whenever a value is assigned to a
//! dereferenced iterator, calls the given callable with the index that corresponds to the offset of the dereferenced
//! iterator and the assigned value.
//!
//! The following code snippet demonstrates how to create a @c tabulate_output_iterator which prints the index and the
//! assigned value.
//!
//! @code
//! #include <cuda/iterator>
//!
//! struct print_op
//! {
//! __host__ __device__ void operator()(int index, float value) const
//! {
//! printf("%d: %f\n", index, value);
//! }
//! };
//!
//! int main()
//! {
//! auto tabulate_it = cuda::make_tabulate_output_iterator(print_op{});
//!
//! tabulate_it[0] = 1.0f; // prints: 0: 1.0
//! tabulate_it[1] = 3.0f; // prints: 1: 3.0
//! tabulate_it[9] = 5.0f; // prints: 9: 5.0
//! }
//! @endcode
template <class _Fn, class _Index>
class tabulate_output_iterator
{
private:
// Not a base because then the friend operators would be ambiguous
::cuda::std::__compressed_movable_box<_Index, _Fn> __store_;
[[nodiscard]] _CCCL_API constexpr _Index& __index() noexcept
{
return __store_.template __get<0>();
}
[[nodiscard]] _CCCL_API constexpr const _Index& __index() const noexcept
{
return __store_.template __get<0>();
}
[[nodiscard]] _CCCL_API constexpr _Fn& __func() noexcept
{
return __store_.template __get<1>();
}
[[nodiscard]] _CCCL_API constexpr const _Fn& __func() const noexcept
{
return __store_.template __get<1>();
}
static_assert(::cuda::std::is_signed_v<_Index>, "tabulate_output_iterator: _Index must be a signed integer");
public:
using iterator_concept = ::cuda::std::random_access_iterator_tag;
using iterator_category = ::cuda::std::random_access_iterator_tag;
using difference_type = _Index;
using value_type = void;
using pointer = void;
using reference = void;
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Fn2 = _Fn)
_CCCL_REQUIRES(::cuda::std::default_initializable<_Fn2>)
_CCCL_API constexpr tabulate_output_iterator() noexcept(::cuda::std::is_nothrow_default_constructible_v<_Fn2>)
: __store_()
{}
//! @brief Constructs a @c tabulate_output_iterator with a given functor and an optional index
//! @param __func the output function
//! @param __index the position in the output sequence
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr tabulate_output_iterator(_Fn __func, _Index __index = 0) noexcept(
::cuda::std::is_nothrow_move_constructible_v<_Fn>)
: __store_(__index, ::cuda::std::move(__func))
{}
//! @brief Returns the stored index
[[nodiscard]] _CCCL_API constexpr difference_type index() const noexcept
{
return __index();
}
//! @brief Dereferences the @c tabulate_output_iterator
//! @returns A proxy that applies the stored function and index on assignment
[[nodiscard]] _CCCL_API constexpr auto operator*() const noexcept
{
return __tabulate_proxy<_Fn, _Index>{const_cast<_Fn&>(__func()), __index()};
}
//! @brief Dereferences the @c tabulate_output_iterator
//! @returns A proxy that applies the stored function and index on assignment
[[nodiscard]] _CCCL_API constexpr auto operator*() noexcept
{
return __tabulate_proxy<_Fn, _Index>{__func(), __index()};
}
//! @brief Subscripts the @c tabulate_output_iterator with a given offset
//! @param __n The additional offset to advance the stored index
//! @returns A proxy that applies the stored function and index on assignment
[[nodiscard]] _CCCL_API constexpr auto operator[](difference_type __n) const noexcept
{
return __tabulate_proxy<_Fn, _Index>{const_cast<_Fn&>(__func()), static_cast<difference_type>(__index() + __n)};
}
//! @brief Subscripts the @c tabulate_output_iterator with a given offset
//! @param __n The additional offset to advance the stored index
//! @returns A proxy that applies the stored function and index on assignment
[[nodiscard]] _CCCL_API constexpr auto operator[](difference_type __n) noexcept
{
return __tabulate_proxy<_Fn, _Index>{__func(), static_cast<difference_type>(__index() + __n)};
}
//! @brief Increments the @c tabulate_output_iterator by incrementing the stored index
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr tabulate_output_iterator& operator++() noexcept
{
++__index();
return *this;
}
//! @brief Increments the @c tabulate_output_iterator by incrementing the stored index
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr tabulate_output_iterator
operator++(int) noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Fn>)
{
tabulate_output_iterator __tmp = *this;
++__index();
return __tmp;
}
//! @brief Decrements the @c tabulate_output_iterator by decrementing the stored index
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr tabulate_output_iterator& operator--() noexcept
{
--__index();
return *this;
}
//! @brief Decrements the @c tabulate_output_iterator by decrementing the stored index
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr tabulate_output_iterator
operator--(int) noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Fn>)
{
tabulate_output_iterator __tmp = *this;
--__index();
return __tmp;
}
//! @brief Returns a copy of this @c tabulate_output_iterator advanced a given number of elements
//! @param __n The number of elements to advance
_CCCL_EXEC_CHECK_DISABLE
template <int = 0> // Template because compiler will complain about non-literal return type if _Fn is not a literal
[[nodiscard]] _CCCL_API friend constexpr tabulate_output_iterator
operator+(const tabulate_output_iterator& __iter, difference_type __n) //
noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Fn>)
{
return tabulate_output_iterator{__iter.__func(), static_cast<difference_type>(__iter.__index() + __n)};
}
//! @brief Returns a copy of a @c tabulate_output_iterator advanced a given number of elements
//! @param __n The number of elements to advance
//! @param __iter The original @c tabulate_output_iterator
_CCCL_EXEC_CHECK_DISABLE
template <int = 0> // Template because compiler will complain about non-literal return type if _Fn is not a literal
[[nodiscard]] _CCCL_API friend constexpr tabulate_output_iterator
operator+(difference_type __n, const tabulate_output_iterator& __iter) //
noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Fn>)
{
return tabulate_output_iterator{__iter.__func(), static_cast<difference_type>(__iter.__index() + __n)};
}
//! @brief Advances the @c tabulate_output_iterator by a given number of elements
//! @param __n The number of elements to advance
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr tabulate_output_iterator& operator+=(difference_type __n) noexcept
{
__index() += __n;
return *this;
}
//! @brief Returns a copy of this @c tabulate_output_iterator decremented a given number of elements
//! @param __n The number of elements to decremented
_CCCL_EXEC_CHECK_DISABLE
template <int = 0> // Template because compiler will complain about non-literal return type if _Fn is not a literal
[[nodiscard]] _CCCL_API friend constexpr tabulate_output_iterator
operator-(const tabulate_output_iterator& __iter, difference_type __n) //
noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Fn>)
{
return tabulate_output_iterator{__iter.__func(), static_cast<difference_type>(__iter.__index() - __n)};
}
//! @brief Returns the distance between two @c tabulate_output_iterator 's
[[nodiscard]] _CCCL_API friend constexpr difference_type
operator-(const tabulate_output_iterator& __lhs, const tabulate_output_iterator& __rhs) noexcept
{
return __lhs.__index() - __rhs.__index();
}
//! @brief Decrements the @c tabulate_output_iterator by a given number of elements
//! @param __n The number of elements to decrement
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr tabulate_output_iterator& operator-=(difference_type __n) noexcept
{
__index() -= __n;
return *this;
}
//! @brief Compares two @c tabulate_output_iterator for equality by comparing their indices
[[nodiscard]] _CCCL_API friend constexpr bool
operator==(const tabulate_output_iterator& __lhs, const tabulate_output_iterator& __rhs) noexcept
{
return __lhs.__index() == __rhs.__index();
}
#if _CCCL_STD_VER <= 2017
//! @brief Compares two @c tabulate_output_iterator for inequality by comparing their indices
[[nodiscard]] _CCCL_API friend constexpr bool
operator!=(const tabulate_output_iterator& __lhs, const tabulate_output_iterator& __rhs) noexcept
{
return __lhs.__index() != __rhs.__index();
}
#endif // _CCCL_STD_VER <= 2017
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
//! @brief Three-way-compares two @c tabulate_output_iterator by comparing their indices
[[nodiscard]] _CCCL_API friend constexpr strong_ordering
operator<=>(const tabulate_output_iterator& __lhs, const tabulate_output_iterator& __rhs) noexcept
{
return __lhs.__index() <=> __rhs.__index();
}
#else // ^^^ _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() ^^^ / vvv !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR() vvv
//! @brief Compares two @c tabulate_output_iterator for less than by comparing their indices
[[nodiscard]] _CCCL_API friend constexpr bool
operator<(const tabulate_output_iterator& __lhs, const tabulate_output_iterator& __rhs) noexcept
{
return __lhs.__index() < __rhs.__index();
}
//! @brief Compares two @c tabulate_output_iterator for less equal by comparing their indices
[[nodiscard]] _CCCL_API friend constexpr bool
operator<=(const tabulate_output_iterator& __lhs, const tabulate_output_iterator& __rhs) noexcept
{
return __lhs.__index() <= __rhs.__index();
}
//! @brief Compares two @c tabulate_output_iterator for greater than by comparing their indices
[[nodiscard]] _CCCL_API friend constexpr bool
operator>(const tabulate_output_iterator& __lhs, const tabulate_output_iterator& __rhs) noexcept
{
return __lhs.__index() > __rhs.__index();
}
//! @brief Compares two @c tabulate_output_iterator for greater equal by comparing their indices
[[nodiscard]] _CCCL_API friend constexpr bool
operator>=(const tabulate_output_iterator& __lhs, const tabulate_output_iterator& __rhs) noexcept
{
return __lhs.__index() >= __rhs.__index();
}
#endif // !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
};
#ifndef _CCCL_DOXYGEN_INVOKED
template <class _Fn>
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES tabulate_output_iterator(_Fn) -> tabulate_output_iterator<_Fn, ::cuda::std::ptrdiff_t>;
_CCCL_TEMPLATE(class _Fn, class _Index)
_CCCL_REQUIRES(::cuda::std::__integer_like<_Index>)
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES tabulate_output_iterator(_Fn, _Index) -> tabulate_output_iterator<_Fn, _Index>;
#endif // _CCCL_DOXYGEN_INVOKED
//! @brief Creates a @c tabulate_output_iterator from an output function and an optional index.
//! @param __func The output function
//! @param __index The index of the @c tabulate_output_iterator within a range. The default index is @c 0.
//! @return A new @c tabulate_output_iterator with @c __index as the counter.
//! @relates tabulate_output_iterator
_CCCL_TEMPLATE(class _Fn, class _Integer = ::cuda::std::ptrdiff_t)
_CCCL_REQUIRES(::cuda::std::__integer_like<_Integer>)
[[nodiscard]] _CCCL_API constexpr auto make_tabulate_output_iterator(_Fn __func, _Integer __index = 0)
{
return tabulate_output_iterator{::cuda::std::move(__func), __index};
}
//! @}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ITERATOR_TABULATE_OUTPUT_ITERATOR_H

580
qwen3_6_scripts/cccl_preload/include/cuda/__iterator/transform_input_output_iterator.h
View File

@@ -1,580 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ITERATOR_TRANSFORM_INPUT_OUTPUT_ITERATOR_H
#define _CUDA___ITERATOR_TRANSFORM_INPUT_OUTPUT_ITERATOR_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/iterator.h>
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
# include <cuda/std/__compare/three_way_comparable.h>
#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
#include <cuda/std/__concepts/constructible.h>
#include <cuda/std/__concepts/derived_from.h>
#include <cuda/std/__concepts/equality_comparable.h>
#include <cuda/std/__concepts/invocable.h>
#include <cuda/std/__functional/invoke.h>
#include <cuda/std/__iterator/advance.h>
#include <cuda/std/__iterator/concepts.h>
#include <cuda/std/__iterator/distance.h>
#include <cuda/std/__iterator/iterator_traits.h>
#include <cuda/std/__ranges/compressed_movable_box.h>
#include <cuda/std/__ranges/concepts.h>
#include <cuda/std/__ranges/movable_box.h>
#include <cuda/std/__type_traits/conditional.h>
#include <cuda/std/__type_traits/enable_if.h>
#include <cuda/std/__type_traits/is_assignable.h>
#include <cuda/std/__type_traits/is_nothrow_assignable.h>
#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
#include <cuda/std/__type_traits/is_object.h>
#include <cuda/std/__type_traits/is_reference.h>
#include <cuda/std/__type_traits/remove_cvref.h>
#include <cuda/std/__utility/declval.h>
#include <cuda/std/__utility/forward.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <class _InputFn, class _OutputFn, class _Iter>
class __transform_input_output_proxy
{
private:
template <class, class, class>
friend class transform_input_output_iterator;
_Iter __iter_;
_InputFn& __input_func_;
_OutputFn& __output_func_;
using _InputValueType = ::cuda::std::invoke_result_t<_InputFn, ::cuda::std::iter_value_t<_Iter>>;
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr explicit __transform_input_output_proxy(
_Iter __iter,
_InputFn& __input_func,
_OutputFn& __output_func) noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Iter>)
: __iter_(__iter)
, __input_func_(__input_func)
, __output_func_(__output_func)
{}
public:
_CCCL_HIDE_FROM_ABI __transform_input_output_proxy(const __transform_input_output_proxy&) = default;
_CCCL_HIDE_FROM_ABI __transform_input_output_proxy& operator=(const __transform_input_output_proxy&) = default;
_CCCL_HIDE_FROM_ABI __transform_input_output_proxy(__transform_input_output_proxy&&) = default;
_CCCL_HIDE_FROM_ABI __transform_input_output_proxy& operator=(__transform_input_output_proxy&&) = default;
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Arg)
_CCCL_REQUIRES(::cuda::std::is_invocable_v<_OutputFn&, _Arg> //
_CCCL_AND ::cuda::std::is_assignable_v<::cuda::std::iter_reference_t<_Iter>,
::cuda::std::invoke_result_t<_OutputFn&, _Arg>>)
_CCCL_API constexpr __transform_input_output_proxy& operator=(_Arg&& __arg) noexcept(
noexcept(*__iter_ = ::cuda::std::invoke(__output_func_, ::cuda::std::forward<_Arg>(__arg))))
{
*__iter_ = ::cuda::std::invoke(__output_func_, ::cuda::std::forward<_Arg>(__arg));
return *this;
}
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Arg)
_CCCL_REQUIRES(::cuda::std::is_invocable_v<const _OutputFn&, _Arg>
_CCCL_AND ::cuda::std::is_assignable_v<::cuda::std::iter_reference_t<const _Iter>,
::cuda::std::invoke_result_t<const _OutputFn&, _Arg>>)
_CCCL_API constexpr const __transform_input_output_proxy& operator=(_Arg&& __arg) const
noexcept(noexcept(*__iter_ = ::cuda::std::invoke(__output_func_, ::cuda::std::forward<_Arg>(__arg))))
{
*__iter_ = ::cuda::std::invoke(__output_func_, ::cuda::std::forward<_Arg>(__arg));
return *this;
}
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr operator _InputValueType() const noexcept(noexcept(::cuda::std::invoke(__input_func_, *__iter_)))
{
return ::cuda::std::invoke(__input_func_, *__iter_);
}
};
//! @addtogroup iterators
//! @{
//! @brief @c transform_input_output_iterator is a special kind of iterator which applies transform functions when
//! reading from or writing to dereferenced values. This iterator is useful for algorithms that operate on a type that
//! needs to be serialized/deserialized from values in another iterator, avoiding the need to materialize intermediate
//! results in memory. This also enables the transform functions to be fused with the operations that read and write to
//! the `transform_input_output_iterator`.
//!
//! The following code snippet demonstrates how to create a @c transform_input_output_iterator which performs different
//! transformations when reading from and writing to the iterator.
//!
//! @code
//! #include <cuda/iterator>
//! #include <thrust/device_vector.h>
//!
//! int main()
//! {
//! const size_t size = 4;
//! thrust::device_vector<float> v(size);
//!
//! // Write 1.0f, 2.0f, 3.0f, 4.0f to vector
//! thrust::sequence(v.begin(), v.end(), 1);
//!
//! // Iterator that negates read values and writes squared values
//! auto iter = cuda::make_transform_input_output_iterator(v.begin(),
//! ::cuda::std::negate<float>{}, thrust::square<float>{});
//!
//! // Iterator negates values when reading
//! std::cout << iter[0] << " "; // -1.0f;
//! std::cout << iter[1] << " "; // -2.0f;
//! std::cout << iter[2] << " "; // -3.0f;
//! std::cout << iter[3] << "\n"; // -4.0f;
//!
//! // Write 1.0f, 2.0f, 3.0f, 4.0f to iterator
//! thrust::sequence(iter, iter + size, 1);
//!
//! // Values were squared before writing to vector
//! std::cout << v[0] << " "; // 1.0f;
//! std::cout << v[1] << " "; // 4.0f;
//! std::cout << v[2] << " "; // 9.0f;
//! std::cout << v[3] << "\n"; // 16.0f;
//!
//! }
//! @endcode
template <class _InputFn, class _OutputFn, class _Iter>
class transform_input_output_iterator
{
// Not a base because then the friend operators would be ambiguous
::cuda::std::__compressed_movable_box<_Iter, _InputFn, _OutputFn> __store_;
[[nodiscard]] _CCCL_API constexpr _Iter& __iter() noexcept
{
return __store_.template __get<0>();
}
[[nodiscard]] _CCCL_API constexpr const _Iter& __iter() const noexcept
{
return __store_.template __get<0>();
}
[[nodiscard]] _CCCL_API constexpr _InputFn& __input_func() noexcept
{
return __store_.template __get<1>();
}
[[nodiscard]] _CCCL_API constexpr const _InputFn& __input_func() const noexcept
{
return __store_.template __get<1>();
}
[[nodiscard]] _CCCL_API constexpr _OutputFn& __output_func() noexcept
{
return __store_.template __get<2>();
}
[[nodiscard]] _CCCL_API constexpr const _OutputFn& __output_func() const noexcept
{
return __store_.template __get<2>();
}
public:
using iterator_concept = ::cuda::std::conditional_t<
::cuda::std::__has_random_access_traversal<_Iter>,
::cuda::std::random_access_iterator_tag,
::cuda::std::conditional_t<::cuda::std::__has_bidirectional_traversal<_Iter>,
::cuda::std::bidirectional_iterator_tag,
::cuda::std::conditional_t<::cuda::std::__has_forward_traversal<_Iter>,
::cuda::std::forward_iterator_tag,
::cuda::std::output_iterator_tag>>>;
using iterator_category = ::cuda::std::output_iterator_tag;
using difference_type = ::cuda::std::iter_difference_t<_Iter>;
using value_type = ::cuda::std::invoke_result_t<_InputFn&, ::cuda::std::iter_reference_t<_Iter>>;
using pointer = void;
using reference = __transform_input_output_proxy<_InputFn, _OutputFn, _Iter>;
static_assert(::cuda::std::is_object_v<_InputFn>,
"cuda::transform_input_output_iterator requires that _InputFn is a function object");
static_assert(::cuda::std::is_object_v<_OutputFn>,
"cuda::transform_input_output_iterator requires that _OutputFn is a function object");
static_assert(::cuda::std::__has_forward_traversal<_Iter> || ::cuda::std::output_iterator<_Iter, value_type>,
"cuda::transform_input_output_iterator requires that _Iter models forward_iterator or output_iterator");
static_assert(::cuda::std::is_invocable_v<_InputFn&, ::cuda::std::iter_reference_t<_Iter>>,
"cuda::transform_input_output_iterator requires that _InputFn is invocable on the result of "
"dereferencing _Iter");
//! @brief Default constructs a @c transform_input_output_iterator with a value initialized iterator and functors
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter, class _InputFn2 = _InputFn, class _OutputFn2 = _OutputFn)
_CCCL_REQUIRES(::cuda::std::default_initializable<_Iter2> _CCCL_AND ::cuda::std::default_initializable<_InputFn2>
_CCCL_AND ::cuda::std::default_initializable<_OutputFn2>)
_CCCL_API constexpr transform_input_output_iterator() noexcept(
::cuda::std::is_nothrow_default_constructible_v<_Iter2>
&& ::cuda::std::is_nothrow_default_constructible_v<_InputFn2>
&& ::cuda::std::is_nothrow_default_constructible_v<_OutputFn2>)
: __store_()
{}
//! @brief Constructs a @c transform_input_output_iterator with base iterator, input functor and output functor
//! @param __iter The iterator to transform
//! @param __input_func The input functor to apply to the iterator when reading
//! @param __output_func The output functor to apply to the iterator when writing
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr transform_input_output_iterator(_Iter __iter, _InputFn __input_func, _OutputFn __output_func) //
noexcept(::cuda::std::is_nothrow_move_constructible_v<_Iter>
&& ::cuda::std::is_nothrow_move_constructible_v<_InputFn>
&& ::cuda::std::is_nothrow_move_constructible_v<_OutputFn>)
: __store_(::cuda::std::move(__iter), ::cuda::std::move(__input_func), ::cuda::std::move(__output_func))
{}
//! @brief Returns a const reference to the base iterator stored
[[nodiscard]] _CCCL_API constexpr const _Iter& base() const& noexcept
{
return __iter();
}
//! @brief Extracts the stored base iterator
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr _Iter base() && noexcept(::cuda::std::is_nothrow_move_constructible_v<_Iter>)
{
return ::cuda::std::move(__iter());
}
//! @brief Dereferences the @c transform_input_output_iterator. Returns a proxy that transforms values read from the
//! stored iterator via the stored input functor and transforms assigned values via the output functor
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr reference operator*() const
noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Iter>)
{
return __transform_input_output_proxy{
__iter(), const_cast<_InputFn&>(__input_func()), const_cast<_OutputFn&>(__output_func())};
}
//! @brief Dereferences the @c transform_input_output_iterator. Returns a proxy that transforms values read from the
//! stored iterator via the stored input functor and transforms assigned values via the output functor
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr reference operator*() noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Iter>)
{
return __transform_input_output_proxy{__iter(), __input_func(), __output_func()};
}
//! @brief Subscripts the @c transform_input_output_iterator. Returns a proxy that transforms values read from the
//! stored iterator adbanvd by a given number of elements via the stored input functor and transforms assigned values
//! via the output functor
//! @param __n The number of elements to advance
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__has_random_access_traversal<_Iter2>)
[[nodiscard]] _CCCL_API constexpr reference operator[](difference_type __n) const
noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Iter2>
&& noexcept(::cuda::std::declval<const _Iter2&>() + __n))
{
return __transform_input_output_proxy{
__iter() + __n, const_cast<_InputFn&>(__input_func()), const_cast<_OutputFn&>(__output_func())};
}
//! @brief Subscripts the @c transform_input_output_iterator. Returns a proxy that transforms values read from the
//! stored iterator adbanvd by a given number of elements via the stored input functor and transforms assigned values
//! via the output functor
//! @param __n The number of elements to advance
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__has_random_access_traversal<_Iter2>)
[[nodiscard]] _CCCL_API constexpr reference operator[](difference_type __n) noexcept(
::cuda::std::is_nothrow_copy_constructible_v<_Iter2> && noexcept(::cuda::std::declval<const _Iter2&>() + __n))
{
return __transform_input_output_proxy{__iter() + __n, __input_func(), __output_func()};
}
//! @brief Increments the stored iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr transform_input_output_iterator& operator++() noexcept(noexcept(++::cuda::std::declval<_Iter&>()))
{
++__iter();
return *this;
}
//! @brief Increments the stored iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr transform_input_output_iterator operator++(int) noexcept(
noexcept(++::cuda::std::declval<_Iter&>())
&& ::cuda::std::is_nothrow_copy_constructible_v<_Iter> && ::cuda::std::is_nothrow_copy_constructible_v<_InputFn>
&& ::cuda::std::is_nothrow_copy_constructible_v<_OutputFn>)
{
auto __tmp = *this;
++*this;
return __tmp;
}
//! @brief Decrements the stored iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__has_bidirectional_traversal<_Iter2>)
_CCCL_API constexpr transform_input_output_iterator& operator--() noexcept(noexcept(--::cuda::std::declval<_Iter2&>()))
{
--__iter();
return *this;
}
//! @brief Decrements the stored iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__has_bidirectional_traversal<_Iter2>)
_CCCL_API constexpr transform_input_output_iterator operator--(int) noexcept(
::cuda::std::is_nothrow_copy_constructible_v<_Iter> && noexcept(--::cuda::std::declval<_Iter2&>()))
{
auto __tmp = *this;
--*this;
return __tmp;
}
//! @brief Advances the @c transform_input_output_iterator by a given number of elements
//! @param __n The number of elements to advance
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__has_random_access_traversal<_Iter2>)
_CCCL_API constexpr transform_input_output_iterator&
operator+=(difference_type __n) noexcept(noexcept(::cuda::std::declval<_Iter2&>() += __n))
{
__iter() += __n;
return *this;
}
//! @brief Returns a copy of a @c transform_input_output_iterator advanced by a given number of elements
//! @param __iter The @c transform_input_output_iterator to advance
//! @param __n The number of elements to advance
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator+(const transform_input_output_iterator& __iter, difference_type __n) //
noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Iter>
&& noexcept(::cuda::std::declval<const _Iter2&>() + difference_type{}))
_CCCL_TRAILING_REQUIRES(transform_input_output_iterator)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return transform_input_output_iterator{__iter.__iter() + __n, __iter.__input_func(), __iter.__output_func()};
}
//! @brief Returns a copy of a @c transform_input_output_iterator advanced by a given number of elements
//! @param __n The number of elements to advance
//! @param __iter The @c transform_input_output_iterator to advance
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator+(difference_type __n, const transform_input_output_iterator& __iter) noexcept(
::cuda::std::is_nothrow_copy_constructible_v<_Iter>
&& noexcept(::cuda::std::declval<const _Iter2&>() + difference_type{}))
_CCCL_TRAILING_REQUIRES(transform_input_output_iterator)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return transform_input_output_iterator{__iter.__iter() + __n, __iter.__input_func(), __iter.__output_func()};
}
//! @brief Decrements the @c transform_input_output_iterator by a given number of elements
//! @param __n The number of elements to decrement
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__has_random_access_traversal<_Iter2>)
_CCCL_API constexpr transform_input_output_iterator&
operator-=(difference_type __n) noexcept(noexcept(::cuda::std::declval<_Iter2&>() -= __n))
{
__iter() -= __n;
return *this;
}
//! @brief Returns a copy of a @c transform_input_output_iterator decremented by a given number of elements
//! @param __iter The @c transform_input_output_iterator to decrement
//! @param __n The number of elements to decrement
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator-(const transform_input_output_iterator& __iter, difference_type __n) //
noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Iter>
&& noexcept(::cuda::std::declval<const _Iter2&>() - difference_type{}))
_CCCL_TRAILING_REQUIRES(transform_input_output_iterator)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return transform_input_output_iterator{__iter.__iter() - __n, __iter.__input_func(), __iter.__output_func()};
}
template <class _Iter2>
static constexpr bool __can_difference =
(::cuda::std::__has_random_access_traversal<_Iter2> || ::cuda::std::sized_sentinel_for<_Iter2, _Iter2>);
template <class _Iter2>
static constexpr bool __noexcept_difference =
noexcept(::cuda::std::declval<const _Iter2&>() - ::cuda::std::declval<const _Iter2&>());
//! @brief Returns the distance between two @c transform_input_output_iterator
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto operator-(const transform_input_output_iterator& __lhs,
const transform_input_output_iterator& __rhs) //
noexcept(__noexcept_difference<_Iter2>) _CCCL_TRAILING_REQUIRES(difference_type)(__can_difference<_Iter2>)
{
return __lhs.__iter() - __rhs.__iter();
}
//! @brief Compares two @c transform_input_output_iterator for equality by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator==(const transform_input_output_iterator& __lhs, const transform_input_output_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() == ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::equality_comparable<_Iter2>)
{
return __lhs.__iter() == __rhs.__iter();
}
#if _CCCL_STD_VER <= 2017
//! @brief Compares two @c transform_input_output_iterator for inequality by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator!=(const transform_input_output_iterator& __lhs, const transform_input_output_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() != ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::equality_comparable<_Iter2>)
{
return __lhs.__iter() != __rhs.__iter();
}
#endif // _CCCL_STD_VER <= 2017
//! @brief Compares two @c transform_input_output_iterator for less than by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator<(const transform_input_output_iterator& __lhs, const transform_input_output_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() < ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return __lhs.__iter() < __rhs.__iter();
}
//! @brief Compares two @c transform_input_output_iterator for greater than by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator>(const transform_input_output_iterator& __lhs, const transform_input_output_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() < ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return __lhs.__iter() > __rhs.__iter();
}
//! @brief Compares two @c transform_input_output_iterator for less equal by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator<=(const transform_input_output_iterator& __lhs, const transform_input_output_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() < ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return __lhs.__iter() <= __rhs.__iter();
}
//! @brief Compares two @c transform_input_output_iterator for greater equal by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator>=(const transform_input_output_iterator& __lhs, const transform_input_output_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() < ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return __lhs.__iter() >= __rhs.__iter();
}
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
//! @brief Three-way-compares two @c transform_input_output_iterator, directly three-way-comparing the stored
//! iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator<=>(const transform_input_output_iterator& __lhs, const transform_input_output_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() <=> ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(
::cuda::std::__has_random_access_traversal<_Iter2>&& ::cuda::std::three_way_comparable<_Iter2>)
{
return __lhs.__iter() <=> __rhs.__iter();
}
#endif // !_LIBCUDACXX_HAS_NO_SPACESHIP_OPERATOR
};
//! @brief make_transform_output_iterator creates a @c transform_input_output_iterator from an iterator, an input
//! functor and an output functor
//! @param __iter The iterator pointing to the input range of the newly created @c transform_input_output_iterator.
//! @param __input_fun The input functor used to transform the range when read
//! @param __output_fun The output functor used to transform the range when written
//! @relates transform_input_output_iterator
template <class _InputFn, class _OutputFn, class _Iter>
[[nodiscard]] _CCCL_API constexpr auto
make_transform_input_output_iterator(_Iter __iter, _InputFn __input_fun, _OutputFn __output_fun)
{
return transform_input_output_iterator<_InputFn, _OutputFn, _Iter>{__iter, __input_fun, __output_fun};
}
//! @}
_CCCL_END_NAMESPACE_CUDA
#ifndef _CCCL_DOXYGEN_INVOKED
# if _CCCL_HAS_HOST_STD_LIB()
_CCCL_BEGIN_NAMESPACE_STD
//! transform_input_output_iterator is a C++20 iterator, so it does not play well with legacy STL features like
//! std::distance. To work around that specialize those functions for transform_input_output_iterator
template <class _Diff, class _InputFn, class _OutputFn, class _Iter>
_CCCL_HOST_API constexpr void
advance(::cuda::transform_input_output_iterator<_InputFn, _OutputFn, _Iter>& __iter, _Diff __diff)
{
::cuda::std::advance(__iter, ::cuda::std::move(__diff));
}
template <class _InputFn, class _OutputFn, class _Iter>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::std::iter_difference_t<_Iter>
distance(::cuda::transform_input_output_iterator<_InputFn, _OutputFn, _Iter> __first,
::cuda::transform_input_output_iterator<_InputFn, _OutputFn, _Iter> __last)
{
return ::cuda::std::distance(::cuda::std::move(__first), ::cuda::std::move(__last));
}
template <class _InputFn, class _OutputFn, class _Iter>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::transform_input_output_iterator<_InputFn, _OutputFn, _Iter>
next(::cuda::transform_input_output_iterator<_InputFn, _OutputFn, _Iter> __iter,
::cuda::std::iter_difference_t<_Iter> __n = 1)
{
_CCCL_ASSERT(__n >= 0 || ::cuda::std::__has_bidirectional_traversal<_Iter>,
"Attempt to std::next(it, n) with negative n on a non-bidirectional iterator");
::cuda::std::advance(__iter, __n);
return __iter;
}
template <class _InputFn, class _OutputFn, class _Iter>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::transform_input_output_iterator<_InputFn, _OutputFn, _Iter>
prev(::cuda::transform_input_output_iterator<_InputFn, _OutputFn, _Iter> __iter,
::cuda::std::iter_difference_t<_Iter> __n = 1)
{
_CCCL_ASSERT(__n <= 0 || ::cuda::std::__has_bidirectional_traversal<_Iter>,
"Attempt to std::prev(it, +n) on a non-bidi iterator");
::cuda::std::advance(__iter, -__n);
return __iter;
}
_CCCL_END_NAMESPACE_STD
# endif // _CCCL_HAS_HOST_STD_LIB()
#endif // _CCCL_DOXYGEN_INVOKED
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ITERATOR_TRANSFORM_INPUT_OUTPUT_ITERATOR_H

582
qwen3_6_scripts/cccl_preload/include/cuda/__iterator/transform_iterator.h
View File

@@ -1,582 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ITERATOR_TRANSFORM_ITERATOR_H
#define _CUDA___ITERATOR_TRANSFORM_ITERATOR_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/iterator.h>
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
# include <cuda/std/__compare/three_way_comparable.h>
#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
#include <cuda/std/__concepts/constructible.h>
#include <cuda/std/__concepts/derived_from.h>
#include <cuda/std/__concepts/equality_comparable.h>
#include <cuda/std/__concepts/invocable.h>
#include <cuda/std/__functional/invoke.h>
#include <cuda/std/__iterator/advance.h>
#include <cuda/std/__iterator/concepts.h>
#include <cuda/std/__iterator/distance.h>
#include <cuda/std/__iterator/iterator_traits.h>
#include <cuda/std/__ranges/compressed_movable_box.h>
#include <cuda/std/__ranges/concepts.h>
#include <cuda/std/__ranges/movable_box.h>
#include <cuda/std/__type_traits/conditional.h>
#include <cuda/std/__type_traits/enable_if.h>
#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
#include <cuda/std/__type_traits/is_object.h>
#include <cuda/std/__type_traits/is_reference.h>
#include <cuda/std/__type_traits/remove_cvref.h>
#include <cuda/std/__utility/declval.h>
#include <cuda/std/__utility/forward.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @addtogroup iterators
//! @{
template <class, class, class = void>
struct __transform_iterator_category_base
{};
template <class _Fn, class _Iter>
struct __transform_iterator_category_base<_Fn,
_Iter,
::cuda::std::enable_if_t<::cuda::std::__has_forward_traversal<_Iter>>>
{
using _Cat = typename ::cuda::std::iterator_traits<_Iter>::iterator_category;
using iterator_category = ::cuda::std::conditional_t<
::cuda::std::is_reference_v<::cuda::std::invoke_result_t<_Fn&, ::cuda::std::iter_reference_t<_Iter>>>,
::cuda::std::conditional_t<::cuda::std::derived_from<_Cat, ::cuda::std::contiguous_iterator_tag>,
::cuda::std::random_access_iterator_tag,
_Cat>,
::cuda::std::input_iterator_tag>;
};
template <class _Fn, class _Iter, bool = (::cuda::std::__has_random_access_traversal<_Iter>)>
inline constexpr bool __transform_iterator_nothrow_subscript = false;
template <class _Fn, class _Iter>
inline constexpr bool __transform_iterator_nothrow_subscript<_Fn, _Iter, true> =
noexcept(::cuda::std::invoke(::cuda::std::declval<_Fn&>(), ::cuda::std::declval<_Iter&>()[0]));
//! @brief @c transform_iterator is an iterator which represents a pointer into a range of values after transformation
//! by a functor. This iterator is useful for creating a range filled with the result of applying an operation to
//! another range without either explicitly storing it in memory, or explicitly executing the transformation. Using
//! @c transform_iterator facilitates kernel fusion by deferring the execution of a transformation until the value is
//! needed while saving both memory capacity and bandwidth.
//!
//! The following code snippet demonstrates how to create a @c transform_iterator which represents the result of
//! @c sqrtf applied to the contents of a @c thrust::device_vector.
//!
//! @code
//! #include <cuda/iterator>
//! #include <thrust/device_vector.h>
//!
//! struct square_root
//! {
//! __host__ __device__
//! float operator()(float x) const
//! {
//! return sqrtf(x);
//! }
//! };
//!
//! int main()
//! {
//! thrust::device_vector<float> v{1.0f, 4.0f, 9.0f, 16.0f};
//!
//! using FloatIterator = thrust::device_vector<float>::iterator;
//!
//! cuda::transform_iterator iter(v.begin(), square_root{});
//!
//! *iter; // returns 1.0f
//! iter[0]; // returns 1.0f;
//! iter[1]; // returns 2.0f;
//! iter[2]; // returns 3.0f;
//! iter[3]; // returns 4.0f;
//!
//! // iter[4] is an out-of-bounds error
//! }
//! @endcode
//!
//! This next example demonstrates how to use a @c transform_iterator with the @c thrust::reduce functor to compute the
//! sum of squares of a sequence. We will create temporary @c transform_iterators utilising class template argument
//! deduction avoid explicitly specifying their type:
//!
//! @code
//! #include <cuda/iterator>
//! #include <thrust/device_vector.h>
//! #include <thrust/reduce.h>
//! #include <iostream>
//!
//! struct square
//! {
//! __host__ __device__
//! float operator()(float x) const
//! {
//! return x * x;
//! }
//! };
//!
//! int main()
//! {
//! // initialize a device array
//! thrust::device_vector<float> v(4);
//! v[0] = 1.0f;
//! v[1] = 2.0f;
//! v[2] = 3.0f;
//! thrust::device_vector<float> v{1.0f, 2.0f, 3.0f, 4.0f};
//! thrust::reduce(cuda::transform_iterator{v.begin(), square{}},
//! cuda::transform_iterator{v.end(), square{}});
//!
//! std::cout << "sum of squares: " << sum_of_squares << '\n';
//! return 0;
//! }
//! @endcode
template <class _Fn, class _Iter>
class transform_iterator : public __transform_iterator_category_base<_Fn, _Iter>
{
static_assert(::cuda::std::is_object_v<_Fn>, "cuda::transform_iterator requires that _Fn is a functor object");
static_assert(::cuda::std::regular_invocable<_Fn&, ::cuda::std::iter_reference_t<_Iter>>,
"cuda::transform_iterator requires that _Fn is invocable with iter_reference_t<_Iter>");
static_assert(::cuda::std::__can_reference<::cuda::std::invoke_result_t<_Fn&, ::cuda::std::iter_reference_t<_Iter>>>,
"cuda::transform_iterator requires that the return type of _Fn is referenceable");
// Not a base because then the friend operators would be ambiguous
::cuda::std::__compressed_movable_box<_Iter, _Fn> __store_;
[[nodiscard]] _CCCL_API constexpr _Iter& __iter() noexcept
{
return __store_.template __get<0>();
}
[[nodiscard]] _CCCL_API constexpr const _Iter& __iter() const noexcept
{
return __store_.template __get<0>();
}
[[nodiscard]] _CCCL_API constexpr _Fn& __func() noexcept
{
return __store_.template __get<1>();
}
[[nodiscard]] _CCCL_API constexpr const _Fn& __func() const noexcept
{
return __store_.template __get<1>();
}
public:
using iterator_concept = ::cuda::std::conditional_t<
::cuda::std::__has_random_access_traversal<_Iter>,
::cuda::std::random_access_iterator_tag,
::cuda::std::conditional_t<::cuda::std::__has_bidirectional_traversal<_Iter>,
::cuda::std::bidirectional_iterator_tag,
::cuda::std::conditional_t<::cuda::std::__has_forward_traversal<_Iter>,
::cuda::std::forward_iterator_tag,
::cuda::std::input_iterator_tag>>>;
using value_type =
::cuda::std::remove_cvref_t<::cuda::std::invoke_result_t<_Fn&, ::cuda::std::iter_reference_t<_Iter>>>;
using difference_type = ::cuda::std::iter_difference_t<_Iter>;
// Those are technically not to spec, but pre-ranges iterator_traits do not work properly with iterators that do not
// define all 5 aliases, see https://en.cppreference.com/w/cpp/iterator/iterator_traits.html
using reference = ::cuda::std::invoke_result_t<_Fn&, ::cuda::std::iter_reference_t<_Iter>>;
using pointer = void;
//! @brief Default constructs a @c transform_iterator with a value initialized iterator and functor
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter, class _Fn2 = _Fn)
_CCCL_REQUIRES(::cuda::std::default_initializable<_Iter2> _CCCL_AND ::cuda::std::default_initializable<_Fn2>)
_CCCL_API constexpr transform_iterator() noexcept(
::cuda::std::is_nothrow_default_constructible_v<_Iter2> && ::cuda::std::is_nothrow_default_constructible_v<_Fn2>)
: __store_()
{}
//! @brief Constructs a @c transform_iterator with a given iterator and functor
//! @param __iter The iterator to transform
//! @param __func The functor to apply to the iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr transform_iterator(_Iter __iter, _Fn __func) noexcept(
::cuda::std::is_nothrow_move_constructible_v<_Iter> && ::cuda::std::is_nothrow_move_constructible_v<_Fn>)
: __store_(::cuda::std::move(__iter), ::cuda::std::move(__func))
{}
//! @brief Returns a const reference to the stored iterator
[[nodiscard]] _CCCL_API constexpr const _Iter& base() const& noexcept
{
return __iter();
}
//! @brief Extracts the stored iterator
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr _Iter base() && noexcept(::cuda::std::is_nothrow_move_constructible_v<_Iter>)
{
return ::cuda::std::move(__iter());
}
//! @brief Dereferences the stored iterator and applies the stored functor to the result
_CCCL_EXEC_CHECK_DISABLE
#ifndef _CCCL_DOXYGEN_INVOKED // Doxygen has issues with the constraint
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::regular_invocable<const _Fn&, ::cuda::std::iter_reference_t<const _Iter2>>)
#endif // !_CCCL_DOXYGEN_INVOKED
[[nodiscard]] _CCCL_API constexpr reference operator*() const
noexcept(noexcept(::cuda::std::invoke(::cuda::std::declval<const _Fn&>(), *::cuda::std::declval<const _Iter2&>())))
{
return ::cuda::std::invoke(__func(), *__iter());
}
//! @cond
//! @brief Dereferences the stored iterator and applies the stored functor to the result
//! @note This is a cludge against the fact that the iterator concepts requires `const Iter` but a user might have
//! forgotten to const qualify the call operator
_CCCL_EXEC_CHECK_DISABLE
#ifndef _CCCL_DOXYGEN_INVOKED // Doxygen has issues with the constraint
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES((!::cuda::std::regular_invocable<const _Fn&, ::cuda::std::iter_reference_t<const _Iter2>>) )
#endif // !_CCCL_DOXYGEN_INVOKED
[[nodiscard]] _CCCL_API constexpr reference operator*() const
noexcept(noexcept(::cuda::std::invoke(::cuda::std::declval<_Fn&>(), *::cuda::std::declval<const _Iter2&>())))
{
return ::cuda::std::invoke(const_cast<_Fn&>(__func()), *__iter());
}
//! @endcond
//! @brief Dereferences the stored iterator and applies the stored functor to the result
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr reference
operator*() noexcept(noexcept(::cuda::std::invoke(::cuda::std::declval<_Fn&>(), *::cuda::std::declval<_Iter&>())))
{
return ::cuda::std::invoke(__func(), *__iter());
}
//! @brief Subscripts the stored iterator by a number of elements and applies the stored functor to the result
//! @param __n The number of elements to advance by
_CCCL_EXEC_CHECK_DISABLE
#ifndef _CCCL_DOXYGEN_INVOKED // Doxygen has issues with the constraint
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__has_random_access_traversal<_Iter2>
_CCCL_AND ::cuda::std::regular_invocable<const _Fn&, ::cuda::std::iter_reference_t<const _Iter2>>)
#endif // !_CCCL_DOXYGEN_INVOKED
[[nodiscard]] _CCCL_API constexpr reference operator[](difference_type __n) const
noexcept(__transform_iterator_nothrow_subscript<const _Fn, _Iter2>)
{
return ::cuda::std::invoke(__func(), __iter()[__n]);
}
//! @cond
//! @brief Subscripts the stored iterator by a number of elements and applies the stored functor to the result
//! @param __n The number of elements to advance by
//! @note This is a cludge against the fact that the iterator concepts requires `const Iter` but a user might have
//! forgotten to const qualify the call operator
_CCCL_EXEC_CHECK_DISABLE
#ifndef _CCCL_DOXYGEN_INVOKED // Doxygen has issues with the constraint
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__has_random_access_traversal<_Iter2> _CCCL_AND(
!::cuda::std::regular_invocable<const _Fn&, ::cuda::std::iter_reference_t<const _Iter2>>))
#endif // !_CCCL_DOXYGEN_INVOKED
[[nodiscard]] _CCCL_API constexpr reference operator[](difference_type __n) const
noexcept(__transform_iterator_nothrow_subscript<_Fn, _Iter2>)
{
return ::cuda::std::invoke(const_cast<_Fn&>(__func()), __iter()[__n]);
}
//! @endcond
//! @brief Subscripts the stored iterator by a number of elements and applies the stored functor to the result
//! @param __n The number of elements to advance by
_CCCL_EXEC_CHECK_DISABLE
#ifndef _CCCL_DOXYGEN_INVOKED // Doxygen has issues with the constraint
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__has_random_access_traversal<_Iter2>)
#endif // !_CCCL_DOXYGEN_INVOKED
[[nodiscard]] _CCCL_API constexpr reference
operator[](difference_type __n) noexcept(__transform_iterator_nothrow_subscript<_Fn, _Iter2>)
{
return ::cuda::std::invoke(__func(), __iter()[__n]);
}
//! @brief Increments the stored iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr transform_iterator& operator++() noexcept(noexcept(++::cuda::std::declval<_Iter&>()))
{
++__iter();
return *this;
}
//! @brief Increments the stored iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr auto operator++(int) noexcept(noexcept(++::cuda::std::declval<_Iter&>()))
{
if constexpr (::cuda::std::__has_forward_traversal<_Iter>)
{
auto __tmp = *this;
++*this;
return __tmp;
}
else
{
++__iter();
}
}
//! @brief Decrements the stored iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__has_bidirectional_traversal<_Iter2>)
_CCCL_API constexpr transform_iterator& operator--() noexcept(noexcept(--::cuda::std::declval<_Iter2&>()))
{
--__iter();
return *this;
}
//! @brief Decrements the stored iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__has_bidirectional_traversal<_Iter2>)
_CCCL_API constexpr transform_iterator operator--(int) noexcept(
::cuda::std::is_nothrow_copy_constructible_v<_Iter> && noexcept(--::cuda::std::declval<_Iter2&>()))
{
auto __tmp = *this;
--*this;
return __tmp;
}
//! @brief Increments the @c transform_iterator by a given number of elements
//! @param __n The number of elements to increment
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__has_random_access_traversal<_Iter2>)
_CCCL_API constexpr transform_iterator&
operator+=(difference_type __n) noexcept(noexcept(::cuda::std::declval<_Iter2&>() += __n))
{
__iter() += __n;
return *this;
}
//! @brief Returns a copy of a @c transform_iterator advanced by a given number of elements
//! @param __iter The @c transform_iterator to advance
//! @param __n The amount of elements to increment
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto operator+(const transform_iterator& __iter, difference_type __n)
_CCCL_TRAILING_REQUIRES(transform_iterator)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return transform_iterator{__iter.__iter() + __n, __iter.__func()};
}
//! @brief Returns a copy of a @c transform_iterator advanced by a given number of elements
//! @param __n The amount of elements to increment
//! @param __iter The @c transform_iterator to advance
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto operator+(difference_type __n, const transform_iterator& __iter)
_CCCL_TRAILING_REQUIRES(transform_iterator)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return transform_iterator{__iter.__iter() + __n, __iter.__func()};
}
//! @brief Decrements the @c transform_iterator by a given number of elements
//! @param __n The number of elements to decrement
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__has_random_access_traversal<_Iter2>)
_CCCL_API constexpr transform_iterator&
operator-=(difference_type __n) noexcept(noexcept(::cuda::std::declval<_Iter2&>() -= __n))
{
__iter() -= __n;
return *this;
}
//! @brief Returns a copy of a @c transform_iterator decremented by a given number of elements
//! @param __iter The @c transform_iterator to decrement
//! @param __n The amount of elements to decrement
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto operator-(const transform_iterator& __iter, difference_type __n)
_CCCL_TRAILING_REQUIRES(transform_iterator)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return transform_iterator{__iter.__iter() - __n, __iter.__func()};
}
template <class _Iter2>
static constexpr bool __can_difference =
(::cuda::std::__has_random_access_traversal<_Iter2> || ::cuda::std::sized_sentinel_for<_Iter2, _Iter2>);
template <class _Iter2>
static constexpr bool __noexcept_difference =
noexcept(::cuda::std::declval<const _Iter2&>() - ::cuda::std::declval<const _Iter2&>());
//! @brief Returns the distance between two @c transform_iterator
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator-(const transform_iterator& __lhs, const transform_iterator& __rhs) noexcept(__noexcept_difference<_Iter2>)
_CCCL_TRAILING_REQUIRES(difference_type)(__can_difference<_Iter2>)
{
return __lhs.__iter() - __rhs.__iter();
}
//! @brief Compares two @c transform_iterator for equality by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator==(const transform_iterator& __lhs, const transform_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() == ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::equality_comparable<_Iter2>)
{
return __lhs.__iter() == __rhs.__iter();
}
#if _CCCL_STD_VER <= 2017
//! @brief Compares two @c transform_iterator for inequality by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator!=(const transform_iterator& __lhs, const transform_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() != ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::equality_comparable<_Iter2>)
{
return __lhs.__iter() != __rhs.__iter();
}
#endif // _CCCL_STD_VER <= 2017
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
//! @brief Three-way-compares two @c transform_iterator, directly three-way-comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator<=>(const transform_iterator& __lhs, const transform_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() <=> ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(
::cuda::std::__has_random_access_traversal<_Iter2>&& ::cuda::std::three_way_comparable<_Iter2>)
{
return __lhs.__iter() <=> __rhs.__iter();
}
#else // ^^^ _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() ^^^ / vvv !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR() vvv
//! @brief Compares two @c transform_iterator for less than by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator<(const transform_iterator& __lhs, const transform_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() < ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return __lhs.__iter() < __rhs.__iter();
}
//! @brief Compares two @c transform_iterator for greater than by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator>(const transform_iterator& __lhs, const transform_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() < ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return __lhs.__iter() > __rhs.__iter();
}
//! @brief Compares two @c transform_iterator for less equal by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator<=(const transform_iterator& __lhs, const transform_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() < ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return __lhs.__iter() <= __rhs.__iter();
}
//! @brief Compares two @c transform_iterator for greater equal by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator>=(const transform_iterator& __lhs, const transform_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() < ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return __lhs.__iter() >= __rhs.__iter();
}
#endif // !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
};
//! @brief Creates a @c transform_iterator from a base iterator and a functor
//! @param __iter The iterator of the input range
//! @param __fun The functor used to transform the input range
//! @relates transform_iterator
template <class _Fn, class _Iter>
[[nodiscard]] _CCCL_API constexpr auto make_transform_iterator(_Iter __iter, _Fn __fun)
{
return transform_iterator<_Fn, _Iter>{__iter, __fun};
}
//! @}
_CCCL_END_NAMESPACE_CUDA
#ifndef _CCCL_DOXYGEN_INVOKED
# if _CCCL_HAS_HOST_STD_LIB()
_CCCL_BEGIN_NAMESPACE_STD
//! transform_iterator is a C++20 iterator, so it does not play well with legacy STL features like std::distance
//! To work around that specialize those functions for transform_iterator
template <class _Diff, class _Fn, class _Iter>
_CCCL_HOST_API constexpr void advance(::cuda::transform_iterator<_Fn, _Iter>& __iter, _Diff __diff)
{
::cuda::std::advance(__iter, ::cuda::std::move(__diff));
}
template <class _Fn, class _Iter>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::std::iter_difference_t<_Iter>
distance(::cuda::transform_iterator<_Fn, _Iter> __first, ::cuda::transform_iterator<_Fn, _Iter> __last)
{
return ::cuda::std::distance(::cuda::std::move(__first), ::cuda::std::move(__last));
}
template <class _Fn, class _Iter>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::transform_iterator<_Fn, _Iter>
next(::cuda::transform_iterator<_Fn, _Iter> __iter, ::cuda::std::iter_difference_t<_Iter> __n = 1)
{
_CCCL_ASSERT(__n >= 0 || ::cuda::std::__has_bidirectional_traversal<_Iter>,
"Attempt to std::next(it, n) with negative n on a non-bidirectional iterator");
::cuda::std::advance(__iter, __n);
return __iter;
}
template <class _Fn, class _Iter>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::transform_iterator<_Fn, _Iter>
prev(::cuda::transform_iterator<_Fn, _Iter> __iter, ::cuda::std::iter_difference_t<_Iter> __n = 1)
{
_CCCL_ASSERT(__n <= 0 || ::cuda::std::__has_bidirectional_traversal<_Iter>,
"Attempt to std::prev(it, +n) on a non-bidi iterator");
::cuda::std::advance(__iter, -__n);
return __iter;
}
_CCCL_END_NAMESPACE_STD
# endif // _CCCL_HAS_HOST_STD_LIB()
#endif // _CCCL_DOXYGEN_INVOKED
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ITERATOR_TRANSFORM_ITERATOR_H

535
qwen3_6_scripts/cccl_preload/include/cuda/__iterator/transform_output_iterator.h
View File

@@ -1,535 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ITERATOR_TRANSFORM_OUTPUT_ITERATOR_H
#define _CUDA___ITERATOR_TRANSFORM_OUTPUT_ITERATOR_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/iterator.h>
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
# include <cuda/std/__compare/three_way_comparable.h>
#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
#include <cuda/std/__concepts/constructible.h>
#include <cuda/std/__concepts/derived_from.h>
#include <cuda/std/__concepts/equality_comparable.h>
#include <cuda/std/__concepts/invocable.h>
#include <cuda/std/__functional/invoke.h>
#include <cuda/std/__iterator/advance.h>
#include <cuda/std/__iterator/concepts.h>
#include <cuda/std/__iterator/distance.h>
#include <cuda/std/__iterator/iterator_traits.h>
#include <cuda/std/__ranges/compressed_movable_box.h>
#include <cuda/std/__ranges/concepts.h>
#include <cuda/std/__ranges/movable_box.h>
#include <cuda/std/__type_traits/conditional.h>
#include <cuda/std/__type_traits/enable_if.h>
#include <cuda/std/__type_traits/is_assignable.h>
#include <cuda/std/__type_traits/is_nothrow_assignable.h>
#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
#include <cuda/std/__type_traits/is_object.h>
#include <cuda/std/__type_traits/is_reference.h>
#include <cuda/std/__type_traits/remove_cvref.h>
#include <cuda/std/__utility/declval.h>
#include <cuda/std/__utility/forward.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @addtogroup iterators
//! @{
template <class _Fn, class _Iter>
class __transform_output_proxy
{
private:
template <class, class>
friend class transform_output_iterator;
_Iter __iter_;
_Fn& __func_;
template <class _MaybeConstFn, class _Arg>
using _Ret = ::cuda::std::invoke_result_t<_MaybeConstFn&, _Arg>;
public:
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr explicit __transform_output_proxy(_Iter __iter, _Fn& __func) noexcept(
::cuda::std::is_nothrow_copy_constructible_v<_Iter>)
: __iter_(__iter)
, __func_(__func)
{}
_CCCL_HIDE_FROM_ABI __transform_output_proxy(const __transform_output_proxy&) = default;
_CCCL_HIDE_FROM_ABI __transform_output_proxy& operator=(const __transform_output_proxy&) = default;
_CCCL_HIDE_FROM_ABI __transform_output_proxy(__transform_output_proxy&&) = default;
_CCCL_HIDE_FROM_ABI __transform_output_proxy& operator=(__transform_output_proxy&&) = default;
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Arg)
_CCCL_REQUIRES(::cuda::std::is_invocable_v<_Fn&, _Arg> //
_CCCL_AND ::cuda::std::is_assignable_v<::cuda::std::iter_reference_t<_Iter>,
::cuda::std::invoke_result_t<_Fn&, _Arg>>)
_CCCL_API constexpr __transform_output_proxy&
operator=(_Arg&& __arg) noexcept(noexcept(*__iter_ = ::cuda::std::invoke(__func_, ::cuda::std::forward<_Arg>(__arg))))
{
*__iter_ = ::cuda::std::invoke(__func_, ::cuda::std::forward<_Arg>(__arg));
return *this;
}
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Arg)
_CCCL_REQUIRES(::cuda::std::is_invocable_v<const _Fn&, _Arg>
_CCCL_AND ::cuda::std::is_assignable_v<::cuda::std::iter_reference_t<const _Iter>,
::cuda::std::invoke_result_t<const _Fn&, _Arg>>)
_CCCL_API constexpr const __transform_output_proxy& operator=(_Arg&& __arg) const
noexcept(noexcept(*__iter_ = ::cuda::std::invoke(__func_, ::cuda::std::forward<_Arg>(__arg))))
{
*__iter_ = ::cuda::std::invoke(__func_, ::cuda::std::forward<_Arg>(__arg));
return *this;
}
};
//! @brief @c transform_output_iterator is a special kind of output iterator which transforms a value written upon
//! dereference. This iterator is useful for transforming an output from algorithms without explicitly storing the
//! intermediate result in the memory and applying subsequent transformation, thereby avoiding wasting memory capacity
//! and bandwidth. Using @c transform_output_iterator facilitates kernel fusion by deferring execution of transformation
//! until the value is written while saving both memory capacity and bandwidth.
//!
//! The following code snippet demonstrated how to create a @c transform_output_iterator which applies @c sqrtf to the
//! assigning value.
//!
//! @code
//! #include <cuda/iterator>
//! #include <thrust/device_vector.h>
//!
//! struct square_root
//! {
//! __host__ __device__
//! float operator()(float x) const
//! {
//! return cuda::std::sqrtf(x);
//! }
//! };
//!
//! int main()
//! {
//! thrust::device_vector<float> v(4);
//! cuda::transform_output_iterator iter(v.begin(), square_root());
//!
//! iter[0] = 1.0f; // stores sqrtf( 1.0f)
//! iter[1] = 4.0f; // stores sqrtf( 4.0f)
//! iter[2] = 9.0f; // stores sqrtf( 9.0f)
//! iter[3] = 16.0f; // stores sqrtf(16.0f)
//! // iter[4] is an out-of-bounds error
//!
//! v[0]; // returns 1.0f;
//! v[1]; // returns 2.0f;
//! v[2]; // returns 3.0f;
//! v[3]; // returns 4.0f;
//!
//! }
//! @endcode
template <class _Fn, class _Iter>
class transform_output_iterator
{
static_assert(::cuda::std::is_object_v<_Fn>,
"cuda::transform_output_iterator requires that _Fn is a function object");
// Not a base because then the friend operators would be ambiguous
::cuda::std::__compressed_movable_box<_Iter, _Fn> __store_;
[[nodiscard]] _CCCL_API constexpr _Iter& __iter() noexcept
{
return __store_.template __get<0>();
}
[[nodiscard]] _CCCL_API constexpr const _Iter& __iter() const noexcept
{
return __store_.template __get<0>();
}
[[nodiscard]] _CCCL_API constexpr _Fn& __func() noexcept
{
return __store_.template __get<1>();
}
[[nodiscard]] _CCCL_API constexpr const _Fn& __func() const noexcept
{
return __store_.template __get<1>();
}
public:
using iterator_concept = ::cuda::std::conditional_t<
::cuda::std::__has_random_access_traversal<_Iter>,
::cuda::std::random_access_iterator_tag,
::cuda::std::conditional_t<::cuda::std::__has_bidirectional_traversal<_Iter>,
::cuda::std::bidirectional_iterator_tag,
::cuda::std::conditional_t<::cuda::std::__has_forward_traversal<_Iter>,
::cuda::std::forward_iterator_tag,
::cuda::std::output_iterator_tag>>>;
using iterator_category = ::cuda::std::output_iterator_tag;
using difference_type = ::cuda::std::iter_difference_t<_Iter>;
using value_type = void;
using pointer = void;
using reference = void;
//! @brief Default constructs a @c transform_output_iterator with a value initialized iterator and functor
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter, class _Fn2 = _Fn)
_CCCL_REQUIRES(::cuda::std::default_initializable<_Iter2> _CCCL_AND ::cuda::std::default_initializable<_Fn2>)
_CCCL_API constexpr transform_output_iterator() noexcept(
::cuda::std::is_nothrow_default_constructible_v<_Iter2> && ::cuda::std::is_nothrow_default_constructible_v<_Fn2>)
: __store_()
{}
//! @brief Constructs a @c transform_output_iterator with a given iterator and output functor
//! @param __iter The iterator to transform
//! @param __func The output function to apply to the iterator on assignment
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr transform_output_iterator(_Iter __iter, _Fn __func) noexcept(
::cuda::std::is_nothrow_move_constructible_v<_Iter> && ::cuda::std::is_nothrow_move_constructible_v<_Fn>)
: __store_(::cuda::std::move(__iter), ::cuda::std::move(__func))
{}
//! @brief Returns a const reference to the stored iterator
[[nodiscard]] _CCCL_API constexpr const _Iter& base() const& noexcept
{
return __iter();
}
//! @brief Extracts the stored iterator
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr _Iter base() && noexcept(::cuda::std::is_nothrow_move_constructible_v<_Iter>)
{
return ::cuda::std::move(__iter());
}
//! @brief Returns a proxy that transforms the input upon assignment
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr auto operator*() const noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Iter>)
{
return __transform_output_proxy{__iter(), const_cast<_Fn&>(__func())};
}
//! @brief Returns a proxy that transforms the input upon assignment
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr auto operator*() noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Iter>)
{
return __transform_output_proxy{__iter(), __func()};
}
//! @brief Subscripts the @c transform_output_iterator
//! @returns A proxy that transforms the input upon assignment storing the current iterator advanced by a given
//! @param __n The number of elements to advance by
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__iter_can_subscript<_Iter2>)
[[nodiscard]] _CCCL_API constexpr auto operator[](difference_type __n) const
noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Iter2>
&& noexcept(::cuda::std::declval<const _Iter2&>() + __n))
{
return __transform_output_proxy{__iter() + __n, const_cast<_Fn&>(__func())};
}
//! @brief Subscripts the @c transform_output_iterator
//! @returns A proxy that transforms the input upon assignment storing the current iterator advanced by a given
//! @param __n The number of elements to advance by
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__iter_can_subscript<_Iter2>)
[[nodiscard]] _CCCL_API constexpr auto operator[](difference_type __n) noexcept(
::cuda::std::is_nothrow_copy_constructible_v<_Iter2> && noexcept(::cuda::std::declval<_Iter2&>() + __n))
{
return __transform_output_proxy{__iter() + __n, const_cast<_Fn&>(__func())};
}
//! @brief Increments the stored iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr transform_output_iterator& operator++() noexcept(noexcept(++::cuda::std::declval<_Iter&>()))
{
++__iter();
return *this;
}
//! @brief Increments the stored iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr auto operator++(int) noexcept(noexcept(++::cuda::std::declval<_Iter&>()))
{
if constexpr (::cuda::std::__has_forward_traversal<_Iter> || ::cuda::std::output_iterator<_Iter, value_type>)
{
auto __tmp = *this;
++*this;
return __tmp;
}
else
{
++__iter();
}
}
//! @brief Decrements the stored iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__iter_can_decrement<_Iter2>)
_CCCL_API constexpr transform_output_iterator& operator--() noexcept(noexcept(--::cuda::std::declval<_Iter2&>()))
{
--__iter();
return *this;
}
//! @brief Decrements the stored iterator
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__iter_can_decrement<_Iter2>)
_CCCL_API constexpr transform_output_iterator operator--(int) noexcept(
::cuda::std::is_nothrow_copy_constructible_v<_Iter> && noexcept(--::cuda::std::declval<_Iter2&>()))
{
auto __tmp = *this;
--*this;
return __tmp;
}
//! @brief Increments the @c transform_output_iterator by a given number of elements
//! @param __n The number of elements to increment
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__iter_can_plus_equal<_Iter2>)
_CCCL_API constexpr transform_output_iterator&
operator+=(difference_type __n) noexcept(noexcept(::cuda::std::declval<_Iter2&>() += __n))
{
__iter() += __n;
return *this;
}
//! @brief Returns a copy of a @c transform_output_iterator incremented by a given number of elements
//! @param __iter The @c transform_output_iterator to increment
//! @param __n The number of elements to increment
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator+(const transform_output_iterator& __iter, difference_type __n) //
noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Iter2>
&& noexcept(::cuda::std::declval<const _Iter2&>() + difference_type{}))
_CCCL_TRAILING_REQUIRES(transform_output_iterator)(::cuda::std::__iter_can_plus<_Iter2>)
{
return transform_output_iterator{__iter.__iter() + __n, __iter.__func()};
}
//! @brief Returns a copy of a @c transform_output_iterator incremented by a given number of elements
//! @param __n The number of elements to increment
//! @param __iter The @c transform_output_iterator to increment
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator+(difference_type __n, const transform_output_iterator& __iter) noexcept(
::cuda::std::is_nothrow_copy_constructible_v<_Iter2>
&& noexcept(::cuda::std::declval<const _Iter2&>() + difference_type{}))
_CCCL_TRAILING_REQUIRES(transform_output_iterator)(::cuda::std::__iter_can_plus<_Iter2>)
{
return transform_output_iterator{__iter.__iter() + __n, __iter.__func()};
}
//! @brief Decrements the @c transform_output_iterator by a given number of elements
//! @param __n The number of elements to decrement
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Iter2 = _Iter)
_CCCL_REQUIRES(::cuda::std::__iter_can_minus_equal<_Iter2>)
_CCCL_API constexpr transform_output_iterator&
operator-=(difference_type __n) noexcept(noexcept(::cuda::std::declval<_Iter2&>() -= __n))
{
__iter() -= __n;
return *this;
}
//! @brief Returns a copy of a @c transform_output_iterator decremented by a given number of elements
//! @param __iter The @c transform_output_iterator to decrement
//! @param __n The number of elements to decrement
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator-(const transform_output_iterator& __iter, difference_type __n) //
noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Iter2>
&& noexcept(::cuda::std::declval<const _Iter2&>() - difference_type{}))
_CCCL_TRAILING_REQUIRES(transform_output_iterator)(::cuda::std::__iter_can_minus<_Iter2>)
{
return transform_output_iterator{__iter.__iter() - __n, __iter.__func()};
}
template <class _Iter2>
static constexpr bool __can_difference =
(::cuda::std::__has_random_access_traversal<_Iter2> || ::cuda::std::sized_sentinel_for<_Iter2, _Iter2>);
template <class _Iter2>
static constexpr bool __noexcept_difference =
noexcept(::cuda::std::declval<const _Iter2&>() - ::cuda::std::declval<const _Iter2&>());
//! @brief Returns the distance between two @c transform_output_iterator
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto operator-(const transform_output_iterator& __lhs,
const transform_output_iterator& __rhs) //
noexcept(__noexcept_difference<_Iter2>) _CCCL_TRAILING_REQUIRES(difference_type)(__can_difference<_Iter2>)
{
return __lhs.__iter() - __rhs.__iter();
}
//! @brief Compares two @c transform_output_iterator for equality by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator==(const transform_output_iterator& __lhs, const transform_output_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() == ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::equality_comparable<_Iter2>)
{
return __lhs.__iter() == __rhs.__iter();
}
#if _CCCL_STD_VER <= 2017
//! @brief Compares two @c transform_output_iterator for inequality by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator!=(const transform_output_iterator& __lhs, const transform_output_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() != ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::equality_comparable<_Iter2>)
{
return __lhs.__iter() != __rhs.__iter();
}
#endif // _CCCL_STD_VER <= 2017
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
//! @brief Three-way-compares two @c transform_output_iterator by three-way-comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator<=>(const transform_output_iterator& __lhs, const transform_output_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() <=> ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(
::cuda::std::__has_random_access_traversal<_Iter2>&& ::cuda::std::three_way_comparable<_Iter2>)
{
return __lhs.__iter() <=> __rhs.__iter();
}
#else // ^^^ _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() ^^^ / vvv !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR() vvv
//! @brief Compares two @c transform_output_iterator for less than by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator<(const transform_output_iterator& __lhs, const transform_output_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() < ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return __lhs.__iter() < __rhs.__iter();
}
//! @brief Compares two @c transform_output_iterator for greater than by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator>(const transform_output_iterator& __lhs, const transform_output_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() < ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return __lhs.__iter() > __rhs.__iter();
}
//! @brief Compares two @c transform_output_iterator for less equal by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator<=(const transform_output_iterator& __lhs, const transform_output_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() < ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return __lhs.__iter() <= __rhs.__iter();
}
//! @brief Compares two @c transform_output_iterator for greater equal by comparing the stored iterators
_CCCL_EXEC_CHECK_DISABLE
template <class _Iter2 = _Iter>
[[nodiscard]] _CCCL_API friend constexpr auto
operator>=(const transform_output_iterator& __lhs, const transform_output_iterator& __rhs) noexcept(
noexcept(::cuda::std::declval<const _Iter2&>() < ::cuda::std::declval<const _Iter2&>()))
_CCCL_TRAILING_REQUIRES(bool)(::cuda::std::__has_random_access_traversal<_Iter2>)
{
return __lhs.__iter() >= __rhs.__iter();
}
#endif // !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
};
//! @brief Creates a @c transform_output_iterator from an iterator and an output function.
//! @param __iter The iterator of the input range
//! @param __fun The output function
//! @relates transform_output_iterator
template <class _Fn, class _Iter>
[[nodiscard]] _CCCL_API constexpr auto make_transform_output_iterator(_Iter __iter, _Fn __fun)
{
return transform_output_iterator<_Fn, _Iter>{__iter, __fun};
}
//! @}
_CCCL_END_NAMESPACE_CUDA
#ifndef _CCCL_DOXYGEN_INVOKED
# if _CCCL_HAS_HOST_STD_LIB()
_CCCL_BEGIN_NAMESPACE_STD
//! transform_output_iterator is a C++20 iterator, so it does not play well with legacy STL features like std::distance
//! To work around that specialize those functions for transform_output_iterator
template <class _Diff, class _Fn, class _Iter>
_CCCL_HOST_API constexpr void advance(::cuda::transform_output_iterator<_Fn, _Iter>& __iter, _Diff __diff)
{
::cuda::std::advance(__iter, ::cuda::std::move(__diff));
}
template <class _Fn, class _Iter>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::std::iter_difference_t<_Iter>
distance(::cuda::transform_output_iterator<_Fn, _Iter> __first, ::cuda::transform_output_iterator<_Fn, _Iter> __last)
{
return ::cuda::std::distance(::cuda::std::move(__first), ::cuda::std::move(__last));
}
template <class _Fn, class _Iter>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::transform_output_iterator<_Fn, _Iter>
next(::cuda::transform_output_iterator<_Fn, _Iter> __iter, ::cuda::std::iter_difference_t<_Iter> __n = 1)
{
_CCCL_ASSERT(__n >= 0 || ::cuda::std::__has_bidirectional_traversal<_Iter>,
"Attempt to std::next(it, n) with negative n on a non-bidirectional iterator");
::cuda::std::advance(__iter, __n);
return __iter;
}
template <class _Fn, class _Iter>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::transform_output_iterator<_Fn, _Iter>
prev(::cuda::transform_output_iterator<_Fn, _Iter> __iter, ::cuda::std::iter_difference_t<_Iter> __n = 1)
{
_CCCL_ASSERT(__n <= 0 || ::cuda::std::__has_bidirectional_traversal<_Iter>,
"Attempt to std::prev(it, +n) on a non-bidi iterator");
::cuda::std::advance(__iter, -__n);
return __iter;
}
_CCCL_END_NAMESPACE_STD
# endif // _CCCL_HAS_HOST_STD_LIB()
#endif // _CCCL_DOXYGEN_INVOKED
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ITERATOR_TRANSFORM_OUTPUT_ITERATOR_H

282
qwen3_6_scripts/cccl_preload/include/cuda/__iterator/zip_common.h
View File

@@ -1,282 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ITERATOR_ZIP_COMMON_H
#define _CUDA___ITERATOR_ZIP_COMMON_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/iterator.h>
#include <cuda/std/__algorithm/ranges_min_element.h>
#include <cuda/std/__concepts/constructible.h>
#include <cuda/std/__fwd/pair.h>
#include <cuda/std/__fwd/tuple.h>
#include <cuda/std/__iterator/concepts.h>
#include <cuda/std/__iterator/iter_move.h>
#include <cuda/std/__iterator/iter_swap.h>
#include <cuda/std/__iterator/iterator_traits.h>
#include <cuda/std/__iterator/readable_traits.h>
#include <cuda/std/__tuple_dir/get.h>
#include <cuda/std/__tuple_dir/tuple_element.h>
#include <cuda/std/__tuple_dir/tuple_size.h>
#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
#include <cuda/std/__type_traits/remove_cvref.h>
#include <cuda/std/__type_traits/remove_reference.h>
#include <cuda/std/__type_traits/void_t.h>
#include <cuda/std/__utility/declval.h>
#include <cuda/std/__utility/integer_sequence.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <class... _Iterators>
struct __zip_iter_constraints
{
static constexpr bool __all_forward = (::cuda::std::__has_forward_traversal<_Iterators> && ...);
static constexpr bool __all_bidirectional = (::cuda::std::__has_bidirectional_traversal<_Iterators> && ...);
static constexpr bool __all_random_access = (::cuda::std::__has_random_access_traversal<_Iterators> && ...);
static constexpr bool __all_equality_comparable = (::cuda::std::equality_comparable<_Iterators> && ...);
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
static constexpr bool __all_three_way_comparable = (::cuda::std::three_way_comparable<_Iterators> && ...);
#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
// Our C++17 iterators sometimes do not satisfy `sized_sentinel_for` but they should all be random_access
static constexpr bool __all_sized_sentinel =
(::cuda::std::sized_sentinel_for<_Iterators, _Iterators> && ...) || __all_random_access;
static constexpr bool __all_nothrow_iter_movable =
(noexcept(::cuda::std::ranges::__iter_move_cpo{}(::cuda::std::declval<const _Iterators&>())) && ...)
&& (::cuda::std::is_nothrow_move_constructible_v<::cuda::std::iter_rvalue_reference_t<_Iterators>> && ...);
static constexpr bool __all_indirectly_swappable = (::cuda::std::indirectly_swappable<_Iterators> && ...);
static constexpr bool __all_noexcept_swappable = (::cuda::std::__noexcept_swappable<_Iterators> && ...);
static constexpr bool __all_nothrow_move_constructible =
(::cuda::std::is_nothrow_move_constructible_v<_Iterators> && ...);
static constexpr bool __all_default_initializable = (::cuda::std::default_initializable<_Iterators> && ...);
static constexpr bool __all_nothrow_default_constructible =
(::cuda::std::is_nothrow_default_constructible_v<_Iterators> && ...);
};
template <class... _Iterators>
[[nodiscard]] _CCCL_API _CCCL_CONSTEVAL auto __get_zip_iterator_concept()
{
using _Constraints = __zip_iter_constraints<_Iterators...>;
if constexpr (_Constraints::__all_random_access)
{
return ::cuda::std::random_access_iterator_tag();
}
else if constexpr (_Constraints::__all_bidirectional)
{
return ::cuda::std::bidirectional_iterator_tag();
}
else if constexpr (_Constraints::__all_forward)
{
return ::cuda::std::forward_iterator_tag();
}
else
{
return ::cuda::std::input_iterator_tag();
}
}
//! @note Not static functions because nvc++ sometimes has issues with class static functions in device code
struct __zip_op_star
{
template <class... _Iterators>
using reference = ::cuda::std::tuple<::cuda::std::iter_reference_t<_Iterators>...>;
_CCCL_EXEC_CHECK_DISABLE
template <class... _Iterators>
[[nodiscard]] _CCCL_API constexpr auto operator()(const _Iterators&... __iters) const
noexcept(noexcept(reference<_Iterators...>{*__iters...}))
{
return reference<_Iterators...>{*__iters...};
}
};
struct __zip_op_increment
{
_CCCL_EXEC_CHECK_DISABLE
template <class... _Iterators>
_CCCL_API constexpr void operator()(_Iterators&... __iters) const noexcept(noexcept(((void) ++__iters, ...)))
{
((void) ++__iters, ...);
}
};
struct __zip_op_decrement
{
_CCCL_EXEC_CHECK_DISABLE
template <class... _Iterators>
_CCCL_API constexpr void operator()(_Iterators&... __iters) const noexcept(noexcept(((void) --__iters, ...)))
{
((void) --__iters, ...);
}
};
struct __zip_iter_move
{
template <class... _Iterators>
using __iter_move_ret = ::cuda::std::tuple<::cuda::std::iter_rvalue_reference_t<_Iterators>...>;
_CCCL_EXEC_CHECK_DISABLE
template <class... _Iterators>
[[nodiscard]] _CCCL_API constexpr auto operator()(const _Iterators&... __iters) const
noexcept(noexcept(__iter_move_ret<_Iterators...>{::cuda::std::ranges::__iter_move_cpo{}(__iters)...}))
{
return __iter_move_ret<_Iterators...>{::cuda::std::ranges::__iter_move_cpo{}(__iters)...};
}
};
struct __zip_op_eq
{
// Extra level of indirection needed because GCC7 and older clang don't allow you to use
// member functions in noexcept() clauses. We also can't use
// __is_cpp17_nothrow_equality_comparable_v because the tuple-like type passed to these
// functions might not implement operator==().
template <class _Tuple1, class _Tuple2, ::cuda::std::size_t... _Indices>
[[nodiscard]] _CCCL_API static constexpr bool
__do_it(const _Tuple1& __tuple1, const _Tuple2& __tuple2, ::cuda::std::index_sequence<_Indices...>) noexcept(
noexcept(((::cuda::std::get<_Indices>(__tuple1) == ::cuda::std::get<_Indices>(__tuple2)) || ...)))
{
return ((::cuda::std::get<_Indices>(__tuple1) == ::cuda::std::get<_Indices>(__tuple2)) || ...);
}
template <class _Tuple1, class _Tuple2, ::cuda::std::size_t... _Indices>
[[nodiscard]] _CCCL_API constexpr bool
operator()(const _Tuple1& __tuple1, const _Tuple2& __tuple2, ::cuda::std::index_sequence<_Indices...> __seq) const
noexcept(noexcept(::cuda::__zip_op_eq::__do_it(__tuple1, __tuple2, __seq)))
{
return ::cuda::__zip_op_eq::__do_it(__tuple1, __tuple2, __seq);
}
template <class _Tuple1, class _Tuple2>
[[nodiscard]] _CCCL_API constexpr bool operator()(const _Tuple1& __tuple1, const _Tuple2& __tuple2) const
noexcept(noexcept(::cuda::__zip_op_eq::__do_it(
__tuple1,
__tuple2,
::cuda::std::make_index_sequence<::cuda::std::tuple_size_v<::cuda::std::remove_cvref_t<_Tuple1>>>{})))
{
return ::cuda::__zip_op_eq::__do_it(
__tuple1,
__tuple2,
::cuda::std::make_index_sequence<::cuda::std::tuple_size_v<::cuda::std::remove_cvref_t<_Tuple1>>>{});
}
};
template <class _Tp, class _Up>
inline constexpr bool __nothrow_distance =
noexcept(::cuda::std::declval<const _Tp&>() - ::cuda::std::declval<const _Up&>());
template <class _Diff>
struct __zip_op_minus
{
struct __op_comp_abs
{
// abs in cstdlib is not constexpr
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API static constexpr _Diff __abs(_Diff __t) noexcept(noexcept(__t < 0 ? -__t : __t))
{
return __t < 0 ? -__t : __t;
}
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr bool operator()(const _Diff& __x, const _Diff& __y) const
noexcept(noexcept(__op_comp_abs::__abs(__x) < __op_comp_abs::__abs(__y)))
{
return __op_comp_abs::__abs(__x) < __op_comp_abs::__abs(__y);
}
};
// Extra level of indirection needed because GCC7 and older clang don't allow you to use
// member functions in noexcept() clauses.
_CCCL_EXEC_CHECK_DISABLE
template <class _Tuple1, class _Tuple2, ::cuda::std::size_t _Zero, ::cuda::std::size_t... _Indices>
[[nodiscard]] _CCCL_API static constexpr _Diff
__do_it(const _Tuple1& __tuple1, const _Tuple2& __tuple2, ::cuda::std::index_sequence<_Zero, _Indices...>) noexcept(
__nothrow_distance<::cuda::std::tuple_element_t<_Zero, _Tuple1>, ::cuda::std::tuple_element_t<_Zero, _Tuple2>>
&& (__nothrow_distance<::cuda::std::tuple_element_t<_Indices, _Tuple1>,
::cuda::std::tuple_element_t<_Indices, _Tuple2>>
&& ...))
{
const _Diff __first = ::cuda::std::get<0>(__tuple1) - ::cuda::std::get<0>(__tuple2);
if (__first == 0)
{
return __first;
}
const _Diff __temp[] = {__first, ::cuda::std::get<_Indices>(__tuple1) - ::cuda::std::get<_Indices>(__tuple2)...};
return *::cuda::std::ranges::__min_element_cpo{}(__temp, __op_comp_abs{});
}
_CCCL_EXEC_CHECK_DISABLE
template <class _Tuple1, class _Tuple2, ::cuda::std::size_t... _Indices>
[[nodiscard]] _CCCL_API constexpr _Diff
operator()(const _Tuple1& __tuple1, const _Tuple2& __tuple2, ::cuda::std::index_sequence<_Indices...> __seq) const
noexcept(noexcept(__zip_op_minus::__do_it(__tuple1, __tuple2, __seq)))
{
return __zip_op_minus::__do_it(__tuple1, __tuple2, __seq);
}
_CCCL_EXEC_CHECK_DISABLE
template <class _Tuple1, class _Tuple2>
[[nodiscard]] _CCCL_API constexpr _Diff operator()(const _Tuple1& __tuple1, const _Tuple2& __tuple2) const
noexcept(noexcept(__zip_op_minus::__do_it(
__tuple1,
__tuple2,
::cuda::std::make_index_sequence<::cuda::std::tuple_size_v<::cuda::std::remove_cvref_t<_Tuple1>>>{})))
{
return __zip_op_minus::__do_it(
__tuple1,
__tuple2,
::cuda::std::make_index_sequence<::cuda::std::tuple_size_v<::cuda::std::remove_cvref_t<_Tuple1>>>{});
}
};
// We need this to make proxy iterators work because those might not have a working `iter_value_t`
template <class _Iter, class = void>
struct __zip_maybe_proxy_helper
{
using reference = decltype(*::cuda::std::declval<_Iter>());
using value_type = ::cuda::std::remove_reference_t<reference>;
};
template <class _Iter>
struct __zip_maybe_proxy_helper<_Iter, ::cuda::std::void_t<::cuda::std::iter_value_t<_Iter>>>
{
using reference = ::cuda::std::iter_reference_t<_Iter>;
using value_type = ::cuda::std::iter_value_t<_Iter>;
};
template <class _Iter>
using __zip_maybe_proxy_reference_t = typename __zip_maybe_proxy_helper<_Iter>::reference;
template <class _Iter>
using __zip_maybe_proxy_value_type_t = typename __zip_maybe_proxy_helper<_Iter>::value_type;
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ITERATOR_ZIP_COMMON_H

125
qwen3_6_scripts/cccl_preload/include/cuda/__iterator/zip_function.h
View File

@@ -1,125 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ITERATOR_ZIP_FUNCTION_H
#define _CUDA___ITERATOR_ZIP_FUNCTION_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/iterator.h>
#include <cuda/std/__concepts/constructible.h>
#include <cuda/std/__functional/invoke.h>
#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
#include <cuda/std/__utility/declval.h>
#include <cuda/std/__utility/forward.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/__utility/pair.h>
#include <cuda/std/tuple>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @addtogroup iterators
//! @{
//! @brief Adaptor that transforms a functor taking arguments of types @c Ts... into one accepting a @c tuple<Ts...>
//! @tparam _Fn The functor to wrap
//! @relates zip_iterator
template <class _Fn>
class zip_function
{
private:
_Fn __fun_;
public:
//! @brief default construct a zip_function
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Fn2 = _Fn)
_CCCL_REQUIRES(::cuda::std::default_initializable<_Fn2>)
_CCCL_API constexpr zip_function() noexcept(::cuda::std::is_nothrow_default_constructible_v<_Fn2>)
: __fun_()
{}
//! @brief construct a zip_function from a functor
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr zip_function(const _Fn& __fun) noexcept(::cuda::std::is_nothrow_copy_constructible_v<_Fn>)
: __fun_(__fun)
{}
//! @brief construct a zip_function from a functor
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr zip_function(_Fn&& __fun) noexcept(::cuda::std::is_nothrow_move_constructible_v<_Fn>)
: __fun_(::cuda::std::move(__fun))
{}
template <class _Fn2, class _Tuple>
static constexpr bool __is_nothrow_invocable =
noexcept(::cuda::std::apply(::cuda::std::declval<_Fn2>(), ::cuda::std::declval<_Tuple>()));
#ifndef _CCCL_DOXYGEN_INVOKED // Doxygen interprets this as a duplicated function
//! @brief Applies a tuple to the stored functor
//! @param __tuple The tuple of arguments to be passed
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Tuple)
_CCCL_REQUIRES(::cuda::std::__can_apply<const _Fn&, _Tuple>)
[[nodiscard]] _CCCL_API constexpr decltype(auto) operator()(_Tuple&& __tuple) const
noexcept(__is_nothrow_invocable<const _Fn&, _Tuple>)
{
return ::cuda::std::apply(__fun_, ::cuda::std::forward<_Tuple>(__tuple));
}
//! @brief Applies a tuple to the stored functor
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Tuple)
_CCCL_REQUIRES(::cuda::std::__can_apply<_Fn&, _Tuple>)
[[nodiscard]] _CCCL_API constexpr decltype(auto)
operator()(_Tuple&& __tuple) noexcept(__is_nothrow_invocable<_Fn&, _Tuple>)
{
return ::cuda::std::apply(__fun_, ::cuda::std::forward<_Tuple>(__tuple));
}
#endif // !_CCCL_DOXYGEN_INVOKED
[[nodiscard]] _CCCL_API constexpr _Fn& __fun() noexcept
{
return __fun_;
}
[[nodiscard]] _CCCL_API constexpr const _Fn& __fun() const noexcept
{
return __fun_;
}
};
//! @brief Creates a @c zip_function from a function
//! @tparam _Fn The functor to wrap
//! @relates zip_iterator
template <class _Fn>
_CCCL_API constexpr zip_function<::cuda::std::decay_t<_Fn>> make_zip_function(_Fn&& __fun)
{
return zip_function<::cuda::std::decay_t<_Fn>>{::cuda::std::forward<_Fn>(__fun)};
}
//! @}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ITERATOR_ZIP_FUNCTION_H

550
qwen3_6_scripts/cccl_preload/include/cuda/__iterator/zip_iterator.h
View File

@@ -1,550 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ITERATOR_ZIP_ITERATOR_H
#define _CUDA___ITERATOR_ZIP_ITERATOR_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/iterator.h>
#include <cuda/std/__algorithm/ranges_min_element.h>
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
# include <cuda/std/__compare/three_way_comparable.h>
#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
#include <cuda/__iterator/zip_common.h>
#include <cuda/std/__concepts/convertible_to.h>
#include <cuda/std/__concepts/equality_comparable.h>
#include <cuda/std/__functional/invoke.h>
#include <cuda/std/__functional/operations.h>
#include <cuda/std/__iterator/advance.h>
#include <cuda/std/__iterator/concepts.h>
#include <cuda/std/__iterator/distance.h>
#include <cuda/std/__iterator/incrementable_traits.h>
#include <cuda/std/__iterator/iter_move.h>
#include <cuda/std/__iterator/iter_swap.h>
#include <cuda/std/__iterator/iterator_traits.h>
#include <cuda/std/__ranges/concepts.h>
#include <cuda/std/__type_traits/common_type.h>
#include <cuda/std/__type_traits/make_unsigned.h>
#include <cuda/std/__type_traits/remove_cvref.h>
#include <cuda/std/__utility/forward.h>
#include <cuda/std/__utility/integer_sequence.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/tuple>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @addtogroup iterators
//! @{
struct __zv_iter_category_base_none
{};
struct __zv_iter_category_base_tag
{
using iterator_category = ::cuda::std::input_iterator_tag;
};
template <class... _Iterators>
using __zv_iter_category_base =
::cuda::std::conditional_t<__zip_iter_constraints<_Iterators...>::__all_forward,
__zv_iter_category_base_tag,
__zv_iter_category_base_none>;
//! @brief @c zip_iterator is an iterator which represents a @c tuple of iterators. This iterator is useful for creating
//! a virtual array of structures while achieving the same performance and bandwidth as the structure of arrays idiom.
//! @c zip_iterator also facilitates kernel fusion by providing a convenient means of amortizing the execution of the
//! same operation over multiple ranges.
//!
//! The following code snippet demonstrates how to create a @c zip_iterator which represents the result of "zipping"
//! multiple ranges together.
//!
//! @code
//! #include <cuda/iterator>
//! #include <thrust/device_vector.h>
//!
//! thrust::device_vector<int> int_v{0, 1, 2};
//! thrust::device_vector<float> float_v{0.0f, 1.0f, 2.0f};
//! thrust::device_vector<char> char_v{'a', 'b', 'c'};
//!
//! cuda::zip_iterator iter{int_v.begin(), float_v.begin(), char_v.begin()};
//!
//! *iter; // returns (0, 0.0f, 'a')
//! iter[0]; // returns (0, 0.0f, 'a')
//! iter[1]; // returns (1, 1.0f, 'b')
//! iter[2]; // returns (2, 2.0f, 'c')
//!
//! cuda::std::get<0>(iter[2]); // returns 2
//! cuda::std::get<1>(iter[0]); // returns 0.0f
//! cuda::std::get<2>(iter[1]); // returns 'b'
//!
//! // iter[3] is an out-of-bounds error
//! @endcode
//!
//! This example shows how to use @c zip_iterator to copy multiple ranges with a single call to @c thrust::copy.
//!
//! @code
//! #include <cuda/iterator>
//! #include <thrust/device_vector.h>
//!
//! int main()
//! {
//! thrust::device_vector<int> int_in{0, 1, 2}, int_out(3);
//! thrust::device_vector<float> float_in{0.0f, 10.0f, 20.0f}, float_out(3);
//!
//! thrust::copy(cuda::zip_iterator{int_in.begin(), float_in.begin()},
//! cuda::zip_iterator{int_in.end(), float_in.end()},
//! cuda::zip_iterator{int_out.begin(),float_out.begin()});
//!
//! // int_out is now [0, 1, 2]
//! // float_out is now [0.0f, 10.0f, 20.0f]
//!
//! return 0;
//! }
//! @endcode
template <class... _Iterators>
class zip_iterator : public __zv_iter_category_base<_Iterators...>
{
::cuda::std::tuple<_Iterators...> __current_;
template <class...>
friend class zip_iterator;
template <class _Fn>
_CCCL_API static constexpr auto
__zip_apply(const _Fn& __fun,
const ::cuda::std::tuple<_Iterators...>& __tuple1,
const ::cuda::std::tuple<_Iterators...>& __tuple2) //
noexcept(noexcept(__fun(__tuple1, __tuple2, ::cuda::std::make_index_sequence<sizeof...(_Iterators)>())))
{
return __fun(__tuple1, __tuple2, ::cuda::std::make_index_sequence<sizeof...(_Iterators)>());
}
public:
//! @brief Default-constructs a @c zip_iterator by defaulting all stored iterators
_CCCL_HIDE_FROM_ABI zip_iterator() = default;
//! @brief Constructs a @c zip_iterator from a tuple of iterators
//! @param __iters A tuple of iterators
_CCCL_API constexpr explicit zip_iterator(::cuda::std::tuple<_Iterators...> __iters)
: __current_(::cuda::std::move(__iters))
{}
//! @brief Constructs a @c zip_iterator from a tuple of iterators
//! @param __iters A tuple of iterators
_CCCL_TEMPLATE(size_t _NumIterators = sizeof...(_Iterators))
_CCCL_REQUIRES((_NumIterators == 2))
_CCCL_API constexpr explicit zip_iterator(::cuda::std::tuple<_Iterators...> __iters)
: __current_(::cuda::std::get<0>(::cuda::std::move(__iters)), ::cuda::std::get<1>(::cuda::std::move(__iters)))
{}
//! @brief Constructs a @c zip_iterator from variadic set of iterators
//! @param __iters The input iterators
_CCCL_API constexpr explicit zip_iterator(_Iterators... __iters)
: __current_(::cuda::std::move(__iters)...)
{}
using iterator_concept = decltype(__get_zip_iterator_concept<_Iterators...>());
using value_type = ::cuda::std::tuple<__zip_maybe_proxy_value_type_t<_Iterators>...>;
using reference = ::cuda::std::tuple<__zip_maybe_proxy_reference_t<_Iterators>...>;
using difference_type = ::cuda::std::common_type_t<::cuda::std::iter_difference_t<_Iterators>...>;
// Those are technically not to spec, but pre-ranges iterator_traits do not work properly with iterators that do not
// define all 5 aliases, see https://en.cppreference.com/w/cpp/iterator/iterator_traits.html
using pointer = void;
template <class... _OtherIters>
static constexpr bool __all_convertible =
(::cuda::std::convertible_to<_OtherIters, _Iterators> && ...)
&& !(::cuda::std::is_same_v<_Iterators, _OtherIters> && ...);
//! @brief Converts a different @c zip_iterator
//! @param __iter The other @c zip_iterator
_CCCL_TEMPLATE(class... _OtherIters)
_CCCL_REQUIRES((sizeof...(_OtherIters) == sizeof...(_Iterators)) _CCCL_AND __all_convertible<_OtherIters...>)
_CCCL_API constexpr zip_iterator(zip_iterator<_OtherIters...> __iter)
: __current_(::cuda::std::move(__iter.__current_))
{}
//! @brief Dereferences the @c zip_iterator
//! @returns A tuple of references obtained by referencing every stored iterator
[[nodiscard]] _CCCL_API constexpr auto operator*() const
noexcept(noexcept(::cuda::std::apply(__zip_op_star{}, __current_)))
{
return ::cuda::std::apply(__zip_op_star{}, __current_);
}
struct __zip_op_index
{
difference_type __n;
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr reference operator()(const _Iterators&... __iters) const
noexcept(noexcept(reference{__iters[::cuda::std::iter_difference_t<_Iterators>(__n)]...}))
{
return reference{__iters[::cuda::std::iter_difference_t<_Iterators>(__n)]...};
}
};
//! @brief Subscripts the @c zip_iterator with an offset
//! @param __n The additional offset
//! @returns A tuple of references obtained by subscripting every stored iterator
_CCCL_TEMPLATE(class _Constraints = __zip_iter_constraints<_Iterators...>)
_CCCL_REQUIRES(_Constraints::__all_random_access)
_CCCL_API constexpr auto operator[](difference_type __n) const
noexcept(noexcept(::cuda::std::apply(__zip_op_index{__n}, __current_)))
{
return ::cuda::std::apply(__zip_op_index{__n}, __current_);
}
//! @brief Increments all stored iterators
_CCCL_API constexpr zip_iterator& operator++() noexcept(noexcept(::cuda::std::apply(__zip_op_increment{}, __current_)))
{
::cuda::std::apply(__zip_op_increment{}, __current_);
return *this;
}
//! @brief Increments all stored iterators
//! @returns A copy of the original @c zip_iterator if possible
_CCCL_API constexpr auto operator++(int)
{
if constexpr (__zip_iter_constraints<_Iterators...>::__all_forward)
{
auto __tmp = *this;
++*this;
return __tmp;
}
else
{
++*this;
}
}
//! @brief Decrements all stored iterators
_CCCL_TEMPLATE(class _Constraints = __zip_iter_constraints<_Iterators...>)
_CCCL_REQUIRES(_Constraints::__all_bidirectional)
_CCCL_API constexpr zip_iterator& operator--() noexcept(noexcept(::cuda::std::apply(__zip_op_decrement{}, __current_)))
{
::cuda::std::apply(__zip_op_decrement{}, __current_);
return *this;
}
//! @brief Decrements all stored iterators
_CCCL_TEMPLATE(class _Constraints = __zip_iter_constraints<_Iterators...>)
_CCCL_REQUIRES(_Constraints::__all_bidirectional)
_CCCL_API constexpr zip_iterator operator--(int)
{
auto __tmp = *this;
--*this;
return __tmp;
}
struct __zip_op_pe
{
difference_type __n;
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr void operator()(_Iterators&... __iters) const
noexcept(noexcept(((void) (__iters += ::cuda::std::iter_difference_t<_Iterators>(__n)), ...)))
{
((void) (__iters += ::cuda::std::iter_difference_t<_Iterators>(__n)), ...);
}
};
//! @brief Increments all stored iterators by a given number of elements
//! @param __n The number of elements to increment
_CCCL_TEMPLATE(class _Constraints = __zip_iter_constraints<_Iterators...>)
_CCCL_REQUIRES(_Constraints::__all_random_access)
_CCCL_API constexpr zip_iterator&
operator+=(difference_type __n) noexcept(noexcept(::cuda::std::apply(__zip_op_pe{__n}, __current_)))
{
::cuda::std::apply(__zip_op_pe{__n}, __current_);
return *this;
}
struct __zip_op_me
{
difference_type __n;
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr void operator()(_Iterators&... __iters) const
noexcept(noexcept(((void) (__iters -= ::cuda::std::iter_difference_t<_Iterators>(__n)), ...)))
{
((void) (__iters -= ::cuda::std::iter_difference_t<_Iterators>(__n)), ...);
}
};
//! @brief Decrements all stored iterators by a given number of elements
//! @param __n The number of elements to decrement
_CCCL_TEMPLATE(class _Constraints = __zip_iter_constraints<_Iterators...>)
_CCCL_REQUIRES(_Constraints::__all_random_access)
_CCCL_API constexpr zip_iterator&
operator-=(difference_type __n) noexcept(noexcept(::cuda::std::apply(__zip_op_me{__n}, __current_)))
{
::cuda::std::apply(__zip_op_me{__n}, __current_);
return *this;
}
//! @brief Returns a copy of a @c zip_iterator incremented by a given number of elements
//! @param __iter The @c zip_iterator to increment
//! @param __n The number of elements to increment
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator+(const zip_iterator& __iter, difference_type __n)
_CCCL_TRAILING_REQUIRES(zip_iterator)(_Constraints::__all_random_access)
{
auto __rhs = __iter;
__rhs += __n;
return __rhs;
}
//! @brief Returns a copy of a @c zip_iterator incremented by a given number of elements
//! @param __n The number of elements to increment
//! @param __iter The @c zip_iterator to increment
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator+(difference_type __n, const zip_iterator& __iter)
_CCCL_TRAILING_REQUIRES(zip_iterator)(_Constraints::__all_random_access)
{
return __iter + __n;
}
//! @brief Returns a copy of a @c zip_iterator decremented by a given number of elements
//! @param __n The number of elements to decrement
//! @param __iter The @c zip_iterator to decrement
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator-(const zip_iterator& __iter, difference_type __n)
_CCCL_TRAILING_REQUIRES(zip_iterator)(_Constraints::__all_random_access)
{
auto __rhs = __iter;
__rhs -= __n;
return __rhs;
}
//! @brief Returns the distance between two @c zip_iterators
//! @returns The minimal distance between any of the stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator-(const zip_iterator& __n, const zip_iterator& __y)
_CCCL_TRAILING_REQUIRES(difference_type)(_Constraints::__all_sized_sentinel)
{
return __zip_apply(__zip_op_minus<difference_type>{}, __n.__current_, __y.__current_);
}
//! @brief Compares two @c zip_iterator for equality by comparing the tuple of stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator==(const zip_iterator& __n, const zip_iterator& __y)
_CCCL_TRAILING_REQUIRES(bool)(_Constraints::__all_equality_comparable)
{
if constexpr (_Constraints::__all_bidirectional)
{
return __n.__current_ == __y.__current_;
}
else
{
return __zip_apply(__zip_op_eq{}, __n.__current_, __y.__current_);
}
}
#if _CCCL_STD_VER <= 2017
//! @brief Compares two @c zip_iterator for inequality by comparing the tuple of stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator!=(const zip_iterator& __n, const zip_iterator& __y)
_CCCL_TRAILING_REQUIRES(bool)(_Constraints::__all_equality_comparable)
{
if constexpr (_Constraints::__all_bidirectional)
{
return __n.__current_ != __y.__current_;
}
else
{
return !__zip_apply(__zip_op_eq{}, __n.__current_, __y.__current_);
}
}
#endif // _CCCL_STD_VER <= 2017
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
//! @brief Three-way compares two @c zip_iterator by comparing the tuple of stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator<=>(const zip_iterator& __n, const zip_iterator& __y)
_CCCL_TRAILING_REQUIRES(bool)(_Constraints::__all_random_access&& _Constraints::__all_three_way_comparable)
{
return __n.__current_ <=> __y.__current_;
}
#else // ^^^ _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() ^^^ / vvv !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR() vvv
//! @brief Compares two @c zip_iterator for less than by comparing the tuple of stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator<(const zip_iterator& __n, const zip_iterator& __y)
_CCCL_TRAILING_REQUIRES(bool)(_Constraints::__all_random_access)
{
return __n.__current_ < __y.__current_;
}
//! @brief Compares two @c zip_iterator for greater than by comparing the tuple of stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator>(const zip_iterator& __n, const zip_iterator& __y)
_CCCL_TRAILING_REQUIRES(bool)(_Constraints::__all_random_access)
{
return __y < __n;
}
//! @brief Compares two @c zip_iterator for less equal by comparing the tuple of stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator<=(const zip_iterator& __n, const zip_iterator& __y)
_CCCL_TRAILING_REQUIRES(bool)(_Constraints::__all_random_access)
{
return !(__y < __n);
}
//! @brief Compares two @c zip_iterator for greater equal by comparing the tuple of stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator>=(const zip_iterator& __n, const zip_iterator& __y)
_CCCL_TRAILING_REQUIRES(bool)(_Constraints::__all_random_access)
{
return !(__n < __y);
}
#endif // !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
//! @brief Applies `iter_move` by applying it to all stored iterators
// MSVC falls over its feet if this is not a template
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto
iter_move(const zip_iterator& __iter) noexcept(_Constraints::__all_nothrow_iter_movable)
{
return ::cuda::std::apply(__zip_iter_move{}, __iter.__current_);
}
struct __zip_op_iter_swap
{
template <size_t... _Indices>
_CCCL_API constexpr void operator()(const ::cuda::std::tuple<_Iterators...>& __iters1,
const ::cuda::std::tuple<_Iterators...>& __iters2,
::cuda::std::index_sequence<_Indices...>) const
noexcept(__zip_iter_constraints<_Iterators...>::__all_noexcept_swappable)
{
(::cuda::std::ranges::__iter_swap_cpo{}(
::cuda::std::get<_Indices>(__iters1), ::cuda::std::get<_Indices>(__iters2)),
...);
}
};
//! @brief Applies `iter_swap` to two @c zip_iterator by applying it to all stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto
iter_swap(const zip_iterator& __lhs, const zip_iterator& __rhs) noexcept(_Constraints::__all_noexcept_swappable)
_CCCL_TRAILING_REQUIRES(void)(_Constraints::__all_indirectly_swappable)
{
return __zip_apply(__zip_op_iter_swap{}, __lhs.__current_, __rhs.__current_);
}
[[nodiscard]] _CCCL_API constexpr ::cuda::std::tuple<_Iterators...>& __iterators() noexcept
{
return __current_;
}
[[nodiscard]] _CCCL_API constexpr const ::cuda::std::tuple<_Iterators...>& __iterators() const noexcept
{
return __current_;
}
};
#ifndef _CCCL_DOXYGEN_INVOKED
template <class... _Iterators>
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES zip_iterator(::cuda::std::tuple<_Iterators...>) -> zip_iterator<_Iterators...>;
template <class... _Iterators>
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES zip_iterator(_Iterators...) -> zip_iterator<_Iterators...>;
#endif // _CCCL_DOXYGEN_INVOKED
//! @brief Creates a @c zip_iterator from a tuple of iterators.
//! @param __t The tuple of iterators to wrap
//! @relates zip_iterator
template <typename... Iterators>
_CCCL_API constexpr zip_iterator<Iterators...> make_zip_iterator(::cuda::std::tuple<Iterators...> __t)
{
return zip_iterator<Iterators...>{::cuda::std::move(__t)};
}
//! @brief Creates a @c zip_iterator from a variadic number of iterators.
//! @param __iters The iterators to wrap
//! @relates zip_iterator
template <typename... Iterators>
_CCCL_API constexpr zip_iterator<Iterators...> make_zip_iterator(Iterators... __iters)
{
return zip_iterator<Iterators...>{::cuda::std::move(__iters)...};
}
//! @}
_CCCL_END_NAMESPACE_CUDA
// GCC and MSVC2019 have issues determining __is_fancy_pointer in C++17 because they fail to instantiate pointer_traits
#if (_CCCL_COMPILER(GCC) || _CCCL_COMPILER(MSVC)) && _CCCL_STD_VER <= 2017
_CCCL_BEGIN_NAMESPACE_CUDA_STD
template <class... _Iterators>
inline constexpr bool __is_fancy_pointer<::cuda::zip_iterator<_Iterators...>> = false;
_CCCL_END_NAMESPACE_CUDA_STD
#endif // _CCCL_COMPILER(MSVC) && _CCCL_STD_VER <= 2017
#ifndef _CCCL_DOXYGEN_INVOKED
# if _CCCL_HAS_HOST_STD_LIB()
_CCCL_BEGIN_NAMESPACE_STD
//! zip_iterator is a C++20 iterator, so it does not play well with legacy STL features like std::distance
//! To work around that specialize those functions for zip_iterator
template <class _Diff, class... _Iterators>
_CCCL_HOST_API constexpr void advance(::cuda::zip_iterator<_Iterators...>& __iter, _Diff __diff)
{
::cuda::std::advance(__iter, ::cuda::std::move(__diff));
}
template <class... _Iterators>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::std::common_type_t<::cuda::std::iter_difference_t<_Iterators>...>
distance(::cuda::zip_iterator<_Iterators...> __first, ::cuda::zip_iterator<_Iterators...> __last)
{
return ::cuda::std::distance(::cuda::std::move(__first), ::cuda::std::move(__last));
}
template <class... _Iterators>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::zip_iterator<_Iterators...>
next(::cuda::zip_iterator<_Iterators...> __iter,
::cuda::std::common_type_t<::cuda::std::iter_difference_t<_Iterators>...> __n = 1)
{
_CCCL_ASSERT(__n >= 0 || ::cuda::__zip_iter_constraints<_Iterators...>::__all_bidirectional,
"Attempt to std::next(it, n) with negative n on a non-bidirectional iterator");
::cuda::std::advance(__iter, __n);
return __iter;
}
template <class... _Iterators>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::zip_iterator<_Iterators...>
prev(::cuda::zip_iterator<_Iterators...> __iter,
::cuda::std::common_type_t<::cuda::std::iter_difference_t<_Iterators>...> __n = 1)
{
_CCCL_ASSERT(__n <= 0 || ::cuda::__zip_iter_constraints<_Iterators...>::__all_bidirectional,
"Attempt to std::prev(it, +n) on a non-bidi iterator");
::cuda::std::advance(__iter, -__n);
return __iter;
}
_CCCL_END_NAMESPACE_STD
# endif // _CCCL_HAS_HOST_STD_LIB()
#endif // _CCCL_DOXYGEN_INVOKED
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ITERATOR_ZIP_ITERATOR_H

594
qwen3_6_scripts/cccl_preload/include/cuda/__iterator/zip_transform_iterator.h
View File

@@ -1,594 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___ITERATOR_ZIP_TRANSFORM_ITERATOR_H
#define _CUDA___ITERATOR_ZIP_TRANSFORM_ITERATOR_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__fwd/iterator.h>
#include <cuda/std/__algorithm/ranges_min_element.h>
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
# include <cuda/std/__compare/three_way_comparable.h>
#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
#include <cuda/__iterator/zip_common.h>
#include <cuda/std/__concepts/constructible.h>
#include <cuda/std/__concepts/convertible_to.h>
#include <cuda/std/__concepts/equality_comparable.h>
#include <cuda/std/__functional/invoke.h>
#include <cuda/std/__functional/operations.h>
#include <cuda/std/__iterator/advance.h>
#include <cuda/std/__iterator/concepts.h>
#include <cuda/std/__iterator/distance.h>
#include <cuda/std/__iterator/incrementable_traits.h>
#include <cuda/std/__iterator/iterator_traits.h>
#include <cuda/std/__ranges/compressed_movable_box.h>
#include <cuda/std/__ranges/concepts.h>
#include <cuda/std/__ranges/movable_box.h>
#include <cuda/std/__type_traits/common_type.h>
#include <cuda/std/__type_traits/make_unsigned.h>
#include <cuda/std/__type_traits/remove_cvref.h>
#include <cuda/std/__utility/forward.h>
#include <cuda/std/__utility/integer_sequence.h>
#include <cuda/std/__utility/move.h>
#include <cuda/std/tuple>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @addtogroup iterators
//! @{
template <class _Fn, class... _Iterators>
[[nodiscard]] _CCCL_API _CCCL_CONSTEVAL auto __get_zip_transform_iterator_category()
{
using _Constraints = __zip_iter_constraints<_Iterators...>;
// NOLINTBEGIN(bugprone-branch-clone)
if constexpr (!::cuda::std::is_reference_v<
::cuda::std::invoke_result_t<_Fn&, ::cuda::std::iter_reference_t<_Iterators>...>>)
{
return ::cuda::std::input_iterator_tag{};
}
else if constexpr (_Constraints::__all_random_access)
{
return ::cuda::std::random_access_iterator_tag{};
}
else if constexpr (_Constraints::__all_bidirectional)
{
return ::cuda::std::bidirectional_iterator_tag{};
}
else if constexpr (_Constraints::__all_forward)
{
return ::cuda::std::forward_iterator_tag{};
}
else
{
return ::cuda::std::input_iterator_tag{};
}
// NOLINTEND(bugprone-branch-clone)
}
//! @brief @c zip_transform_iterator is an iterator which represents the result of a transformation of a set of
//! sequences with a given function. This iterator is useful for creating a range filled with the result of applying an
//! operation to another range without either explicitly storing it in memory, or explicitly executing the
//! transformation. Using @c zip_transform_iterator facilitates kernel fusion by deferring the execution of a
//! transformation until the value is needed while saving both memory capacity and bandwidth.
//!
//! @c zip_transform_iterator is morally equivalent to a combination of transform_iterator and zip_iterator
//!
//! @code{.cpp}
//! template <class Fn, class... Iterators>
//! using zip_transform_iterator = cuda::transform_iterator<cuda::zip_iterator<Iterators...>, cuda::zip_function<Fn>>;
//! @endcode
//!
//! @c zip_transform_iterator has the additional benefit that it does not require an artificial @c zip_function to work
//! and more importantly does not need to materialize the result of dereferencing the stored iterators when passing them
//! to the stored function.
//!
//! The following code snippet demonstrates how to create a @c zip_transform_iterator which represents the result of
//! "zipping" multiple ranges together.
//!
//! @code
//! #include <cuda/iterator>
//! #include <thrust/device_vector.h>
//!
//! struct SumArgs {
//! __host__ __device__ float operator()(float a, float b, float c) const noexcept {
//! return a + b + c;
//! }
//! };
//!
//! thrust::device_vector<float> A{0.f, 1.f, 2.f};
//! thrust::device_vector<float> B{1.f, 2.f, 3.f};
//! thrust::device_vector<float> C{2.f, 3.f, 4.f};
//!
//! cuda::zip_transform_iterator iter{SumArgs{}, A.begin(), B.begin(), C.begin()};
//!
//! *iter; // returns (3.f)
//! iter[0]; // returns (3.f)
//! iter[1]; // returns (6.f)
//! iter[2]; // returns (9.f)
//! // iter[3] is an out-of-bounds error
//! @endcode
//!
//! This example shows how to use @c zip_transform_iterator to copy multiple ranges with a single call to @c
//! thrust::copy.
//!
//! @code
//! #include <cuda/iterator>
//! #include <thrust/device_vector.h>
//!
//! int main()
//! {
//! struct SumArgs {
//! __host__ __device__ float operator()(float a, float b, float c) const noexcept {
//! return a + b + c;
//! }
//! };
//!
//! thrust::device_vector<float> A{0.f, 1.f, 2.f};
//! thrust::device_vector<float> B{1.f, 2.f, 3.f};
//! thrust::device_vector<float> C{2.f, 3.f, 4.f};
//! thrust::device_vector<float> out(3);
//!
//! cuda::zip_transform_iterator iter{SumArgs{}, A.begin(), B.begin(), C.begin()}
//! thrust::copy(iter, iter + 3, out.begin());
//!
//! // out is now [3.0f, 6.0f, 9.0f]
//!
//! return 0;
//! }
//! @endcode
template <class _Fn, class... _Iterators>
class zip_transform_iterator
{
private:
// Not a base because then the friend operators would be ambiguous
::cuda::std::__compressed_movable_box<::cuda::std::tuple<_Iterators...>, _Fn> __store_;
[[nodiscard]] _CCCL_API constexpr ::cuda::std::tuple<_Iterators...>& __iters() noexcept
{
return __store_.template __get<0>();
}
[[nodiscard]] _CCCL_API constexpr const ::cuda::std::tuple<_Iterators...>& __iters() const noexcept
{
return __store_.template __get<0>();
}
[[nodiscard]] _CCCL_API constexpr _Fn& __func() noexcept
{
return __store_.template __get<1>();
}
[[nodiscard]] _CCCL_API constexpr const _Fn& __func() const noexcept
{
return __store_.template __get<1>();
}
template <class, class...>
friend class zip_transform_iterator;
template <class _Op>
_CCCL_API static constexpr auto
__zip_apply(const _Op& __op,
const ::cuda::std::tuple<_Iterators...>& __tuple1,
const ::cuda::std::tuple<_Iterators...>& __tuple2) //
noexcept(noexcept(__op(__tuple1, __tuple2, ::cuda::std::make_index_sequence<sizeof...(_Iterators)>())))
{
return __op(__tuple1, __tuple2, ::cuda::std::make_index_sequence<sizeof...(_Iterators)>());
}
public:
//! @brief Default-constructs a @c zip_transform_iterator by value-initializing the functor and all stored iterators
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Fn2 = _Fn)
_CCCL_REQUIRES(
::cuda::std::default_initializable<_Fn2>&& __zip_iter_constraints<_Iterators...>::__all_default_initializable)
_CCCL_API constexpr zip_transform_iterator() noexcept(
::cuda::std::is_nothrow_default_constructible_v<_Fn2>
&& __zip_iter_constraints<_Iterators...>::__all_nothrow_default_constructible)
: __store_()
{}
//! @brief Constructs a @c zip_transform_iterator from a tuple of iterators
//! @param __fun The functor used to transform dereferenced elements.
//! @param __iters A tuple or pair of iterators
_CCCL_API constexpr explicit zip_transform_iterator(_Fn __fun, ::cuda::std::tuple<_Iterators...> __iters)
: __store_(::cuda::std::move(__iters), ::cuda::std::move(__fun))
{}
//! @brief Constructs a @c zip_transform_iterator from variadic set of iterators
//! @param __fun The functor used to transform dereferenced elements.
//! @param __iters The input iterators
_CCCL_API constexpr explicit zip_transform_iterator(_Fn __fun, _Iterators... __iters)
: __store_(::cuda::std::tuple<_Iterators...>{::cuda::std::move(__iters)...}, ::cuda::std::move(__fun))
{}
using iterator_concept = decltype(::cuda::__get_zip_iterator_concept<_Iterators...>());
using iterator_category = decltype(::cuda::__get_zip_transform_iterator_category<_Fn, _Iterators...>());
using difference_type = ::cuda::std::common_type_t<::cuda::std::iter_difference_t<_Iterators>...>;
using value_type =
::cuda::std::remove_cvref_t<::cuda::std::invoke_result_t<_Fn&, ::cuda::std::iter_reference_t<_Iterators>...>>;
// Those are technically not to spec, but pre-ranges iterator_traits do not work properly with iterators that do not
// define all 5 aliases, see https://en.cppreference.com/w/cpp/iterator/iterator_traits.html
using reference = ::cuda::std::invoke_result_t<_Fn&, ::cuda::std::iter_reference_t<_Iterators>...>;
using pointer = void;
// Internal helper functions to extract internals for device dispatch, must be a tuple for cub_transform_many
[[nodiscard]] _CCCL_API constexpr ::cuda::std::tuple<_Iterators...>
__base() && noexcept(::cuda::std::is_nothrow_move_constructible_v<::cuda::std::tuple<_Iterators...>>)
{
return ::cuda::std::move(__iters());
}
[[nodiscard]] _CCCL_API constexpr _Fn __pred() && noexcept(::cuda::std::is_nothrow_move_constructible_v<_Fn>)
{
return ::cuda::std::move(__func());
}
struct __zip_transform_op_star
{
_Fn& __func_;
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr reference operator()(const _Iterators&... __iters) const
noexcept(::cuda::std::is_nothrow_invocable_v<_Fn&, ::cuda::std::iter_reference_t<const _Iterators>...>)
{
return ::cuda::std::invoke(const_cast<_Fn&>(__func_), *__iters...);
}
};
//! @brief Invokes the stored function with the result of dereferencing the stored iterators
[[nodiscard]] _CCCL_API constexpr reference operator*() const
noexcept(::cuda::std::is_nothrow_invocable_v<_Fn&, ::cuda::std::iter_reference_t<const _Iterators>...>)
{
return ::cuda::std::apply(__zip_transform_op_star{const_cast<_Fn&>(__func())}, __iters());
}
struct __zip_transform_op_subscript
{
difference_type __n_;
_Fn& __func_;
_CCCL_EXEC_CHECK_DISABLE
[[nodiscard]] _CCCL_API constexpr reference operator()(const _Iterators&... __iters) const noexcept(noexcept(
::cuda::std::invoke(const_cast<_Fn&>(__func_), __iters[::cuda::std::iter_difference_t<_Iterators>(__n_)]...)))
{
return ::cuda::std::invoke(
const_cast<_Fn&>(__func_), __iters[::cuda::std::iter_difference_t<_Iterators>(__n_)]...);
}
};
//! @brief Invokes the stored function with the result of dereferencing the stored iterators advanced by an offset
//! @param __n The additional offset
_CCCL_TEMPLATE(class _Constraints = __zip_iter_constraints<_Iterators...>)
_CCCL_REQUIRES(_Constraints::__all_random_access)
_CCCL_API constexpr reference operator[](difference_type __n) const
noexcept(noexcept(::cuda::std::apply(__zip_transform_op_subscript{__n, ::cuda::std::declval<_Fn&>()},
::cuda::std::declval<const ::cuda::std::tuple<_Iterators...>&>())))
{
return ::cuda::std::apply(__zip_transform_op_subscript{__n, const_cast<_Fn&>(__func())}, __iters());
}
//! @brief Increments all stored iterators
_CCCL_API constexpr zip_transform_iterator& operator++() noexcept(
noexcept(::cuda::std::apply(__zip_op_increment{}, ::cuda::std::declval<::cuda::std::tuple<_Iterators...>&>())))
{
::cuda::std::apply(__zip_op_increment{}, __iters());
return *this;
}
//! @brief Increments all stored iterators
//! @returns A copy of the original @c zip_transform_iterator if possible
_CCCL_API constexpr auto operator++(int)
{
if constexpr (__zip_iter_constraints<_Iterators...>::__all_forward)
{
auto __tmp = *this;
++*this;
return __tmp;
}
else
{
++*this;
}
}
//! @brief Decrements all stored iterators
_CCCL_TEMPLATE(class _Constraints = __zip_iter_constraints<_Iterators...>)
_CCCL_REQUIRES(_Constraints::__all_bidirectional)
_CCCL_API constexpr zip_transform_iterator& operator--() noexcept(
noexcept(::cuda::std::apply(__zip_op_decrement{}, ::cuda::std::declval<::cuda::std::tuple<_Iterators...>&>())))
{
::cuda::std::apply(__zip_op_decrement{}, __iters());
return *this;
}
//! @brief Decrements all stored iterators
_CCCL_TEMPLATE(class _Constraints = __zip_iter_constraints<_Iterators...>)
_CCCL_REQUIRES(_Constraints::__all_bidirectional)
_CCCL_API constexpr zip_transform_iterator operator--(int)
{
auto __tmp = *this;
--*this;
return __tmp;
}
struct __zip_op_pe
{
difference_type __n;
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr void operator()(_Iterators&... __iters) const
noexcept(noexcept(((void) (__iters += ::cuda::std::iter_difference_t<_Iterators>(__n)), ...)))
{
((void) (__iters += ::cuda::std::iter_difference_t<_Iterators>(__n)), ...);
}
};
//! @brief Increments all stored iterators by a given number of elements
//! @param __n The number of elements to increment
_CCCL_TEMPLATE(class _Constraints = __zip_iter_constraints<_Iterators...>)
_CCCL_REQUIRES(_Constraints::__all_random_access)
_CCCL_API constexpr zip_transform_iterator& operator+=(difference_type __n) noexcept(
noexcept(::cuda::std::apply(__zip_op_pe{__n}, ::cuda::std::declval<::cuda::std::tuple<_Iterators...>&>())))
{
::cuda::std::apply(__zip_op_pe{__n}, __iters());
return *this;
}
struct __zip_op_me
{
difference_type __n;
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API constexpr void operator()(_Iterators&... __iters) const
noexcept(noexcept(((void) (__iters -= ::cuda::std::iter_difference_t<_Iterators>(__n)), ...)))
{
((void) (__iters -= ::cuda::std::iter_difference_t<_Iterators>(__n)), ...);
}
};
//! @brief Decrements all stored iterators by a given number of elements
//! @param __n The number of elements to decrement
_CCCL_TEMPLATE(class _Constraints = __zip_iter_constraints<_Iterators...>)
_CCCL_REQUIRES(_Constraints::__all_random_access)
_CCCL_API constexpr zip_transform_iterator& operator-=(difference_type __n) noexcept(
noexcept(::cuda::std::apply(__zip_op_me{__n}, ::cuda::std::declval<::cuda::std::tuple<_Iterators...>&>())))
{
::cuda::std::apply(__zip_op_me{__n}, __iters());
return *this;
}
//! @brief Returns a copy of a @c zip_transform_iterator incremented by a given number of elements
//! @param __iter The @c zip_transform_iterator to increment
//! @param __n The number of elements to increment
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator+(const zip_transform_iterator& __iter, difference_type __n)
_CCCL_TRAILING_REQUIRES(zip_transform_iterator)(_Constraints::__all_random_access)
{
auto __rhs = __iter;
__rhs += __n;
return __rhs;
}
//! @brief Returns a copy of a @c zip_transform_iterator incremented by a given number of elements
//! @param __n The number of elements to increment
//! @param __iter The @c zip_transform_iterator to increment
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator+(difference_type __n, const zip_transform_iterator& __iter)
_CCCL_TRAILING_REQUIRES(zip_transform_iterator)(_Constraints::__all_random_access)
{
return __iter + __n;
}
//! @brief Returns a copy of a @c zip_transform_iterator decremented by a given number of elements
//! @param __n The number of elements to decrement
//! @param __iter The @c zip_transform_iterator to decrement
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator-(const zip_transform_iterator& __iter, difference_type __n)
_CCCL_TRAILING_REQUIRES(zip_transform_iterator)(_Constraints::__all_random_access)
{
auto __rhs = __iter;
__rhs -= __n;
return __rhs;
}
//! @brief Returns the distance between two @c zip_transform_iterators
//! @returns The minimal distance between any of the stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator-(const zip_transform_iterator& __n, const zip_transform_iterator& __y)
_CCCL_TRAILING_REQUIRES(difference_type)(_Constraints::__all_sized_sentinel)
{
return __zip_apply(__zip_op_minus<difference_type>{}, __n.__iters(), __y.__iters());
}
//! @brief Compares two @c zip_transform_iterator for equality by comparing the tuple of stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator==(const zip_transform_iterator& __n, const zip_transform_iterator& __y)
_CCCL_TRAILING_REQUIRES(bool)(_Constraints::__all_equality_comparable)
{
if constexpr (_Constraints::__all_bidirectional)
{
return __n.__iters() == __y.__iters();
}
else
{
return __zip_apply(__zip_op_eq{}, __n.__iters(), __y.__iters());
}
}
#if _CCCL_STD_VER <= 2017
//! @brief Compares two @c zip_transform_iterator for inequality by comparing the tuple of stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator!=(const zip_transform_iterator& __n, const zip_transform_iterator& __y)
_CCCL_TRAILING_REQUIRES(bool)(_Constraints::__all_equality_comparable)
{
if constexpr (_Constraints::__all_bidirectional)
{
return __n.__iters() != __y.__iters();
}
else
{
return !__zip_apply(__zip_op_eq{}, __n.__iters(), __y.__iters());
}
}
#endif // _CCCL_STD_VER <= 2017
#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
//! @brief Three-way compares two @c zip_transform_iterator by comparing the tuple of stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator<=>(const zip_transform_iterator& __n, const zip_transform_iterator& __y)
_CCCL_TRAILING_REQUIRES(bool)(_Constraints::__all_random_access&& _Constraints::__all_three_way_comparable)
{
return __n.__iters() <=> __y.__iters();
}
#else // ^^^ _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() ^^^ / vvv !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR() vvv
//! @brief Compares two @c zip_transform_iterator for less than by comparing the tuple of stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator<(const zip_transform_iterator& __n, const zip_transform_iterator& __y)
_CCCL_TRAILING_REQUIRES(bool)(_Constraints::__all_random_access)
{
return __n.__iters() < __y.__iters();
}
//! @brief Compares two @c zip_transform_iterator for greater than by comparing the tuple of stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator>(const zip_transform_iterator& __n, const zip_transform_iterator& __y)
_CCCL_TRAILING_REQUIRES(bool)(_Constraints::__all_random_access)
{
return __y < __n;
}
//! @brief Compares two @c zip_transform_iterator for less equal by comparing the tuple of stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator<=(const zip_transform_iterator& __n, const zip_transform_iterator& __y)
_CCCL_TRAILING_REQUIRES(bool)(_Constraints::__all_random_access)
{
return !(__y < __n);
}
//! @brief Compares two @c zip_transform_iterator for greater equal by comparing the tuple of stored iterators
template <class _Constraints = __zip_iter_constraints<_Iterators...>>
_CCCL_API friend constexpr auto operator>=(const zip_transform_iterator& __n, const zip_transform_iterator& __y)
_CCCL_TRAILING_REQUIRES(bool)(_Constraints::__all_random_access)
{
return !(__n < __y);
}
#endif // !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
};
#ifndef _CCCL_DOXYGEN_INVOKED
template <class _Fn, class... _Iterators>
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES zip_transform_iterator(_Fn, ::cuda::std::tuple<_Iterators...>)
-> zip_transform_iterator<_Fn, _Iterators...>;
template <class _Fn, class... _Iterators>
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES zip_transform_iterator(_Fn, _Iterators...)
-> zip_transform_iterator<_Fn, _Iterators...>;
#endif // _CCCL_DOXYGEN_INVOKED
//! @brief Creates a @c zip_transform_iterator from a tuple of iterators.
//! @param __fun The functor used to transform dereferenced elements.
//! @param __t The tuple of iterators to wrap
//! @relates zip_transform_iterator
template <class _Fn, class... _Iterators>
[[nodiscard]] _CCCL_API constexpr auto
make_zip_transform_iterator(_Fn __fun, ::cuda::std::tuple<_Iterators...> __t) noexcept(
::cuda::std::is_nothrow_move_constructible_v<_Fn>
&& __zip_iter_constraints<_Iterators...>::__all_nothrow_move_constructible)
{
return zip_transform_iterator<_Fn, _Iterators...>{::cuda::std::move(__fun), ::cuda::std::move(__t)};
}
//! @brief Creates a @c zip_transform_iterator from a variadic number of iterators.
//! @param __fun The functor used to transform dereferenced elements.
//! @param __iters The iterators to wrap
//! @relates zip_transform_iterator
template <class _Fn, class... _Iterators>
[[nodiscard]] _CCCL_API constexpr auto make_zip_transform_iterator(_Fn __fun, _Iterators... __iters) noexcept(
::cuda::std::is_nothrow_move_constructible_v<_Fn>
&& __zip_iter_constraints<_Iterators...>::__all_nothrow_move_constructible)
{
return zip_transform_iterator<_Fn, _Iterators...>{::cuda::std::move(__fun), ::cuda::std::move(__iters)...};
}
//! @}
_CCCL_END_NAMESPACE_CUDA
// GCC and MSVC2019 have issues determining __is_fancy_pointer in C++17 because they fail to instantiate pointer_traits
#if (_CCCL_COMPILER(GCC) || _CCCL_COMPILER(MSVC)) && _CCCL_STD_VER <= 2017
_CCCL_BEGIN_NAMESPACE_CUDA_STD
template <class _Fn, class... _Iterators>
inline constexpr bool __is_fancy_pointer<::cuda::zip_transform_iterator<_Fn, _Iterators...>> = false;
_CCCL_END_NAMESPACE_CUDA_STD
#endif // (_CCCL_COMPILER(GCC) || _CCCL_COMPILER(MSVC)) && _CCCL_STD_VER <= 2017
#ifndef _CCCL_DOXYGEN_INVOKED
# if _CCCL_HAS_HOST_STD_LIB()
_CCCL_BEGIN_NAMESPACE_STD
//! zip_transform_iterator is a C++20 iterator, so it does not play well with legacy STL features like std::distance
//! To work around that specialize those functions for zip_transform_iterator
template <class _Diff, class _Fn, class... _Iterators>
_CCCL_HOST_API constexpr void advance(::cuda::zip_transform_iterator<_Fn, _Iterators...>& __iter, _Diff __diff)
{
::cuda::std::advance(__iter, ::cuda::std::move(__diff));
}
template <class _Fn, class... _Iterators>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::std::common_type_t<::cuda::std::iter_difference_t<_Iterators>...>
distance(::cuda::zip_transform_iterator<_Fn, _Iterators...> __first,
::cuda::zip_transform_iterator<_Fn, _Iterators...> __last)
{
return ::cuda::std::distance(::cuda::std::move(__first), ::cuda::std::move(__last));
}
template <class _Fn, class... _Iterators>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::zip_transform_iterator<_Fn, _Iterators...>
next(::cuda::zip_transform_iterator<_Fn, _Iterators...> __iter,
::cuda::std::common_type_t<::cuda::std::iter_difference_t<_Iterators>...> __n = 1)
{
_CCCL_ASSERT(__n >= 0 || ::cuda::__zip_iter_constraints<_Iterators...>::__all_bidirectional,
"Attempt to std::next(it, n) with negative n on a non-bidirectional iterator");
::cuda::std::advance(__iter, __n);
return __iter;
}
template <class _Fn, class... _Iterators>
[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::zip_transform_iterator<_Fn, _Iterators...>
prev(::cuda::zip_transform_iterator<_Fn, _Iterators...> __iter,
::cuda::std::common_type_t<::cuda::std::iter_difference_t<_Iterators>...> __n = 1)
{
_CCCL_ASSERT(__n <= 0 || ::cuda::__zip_iter_constraints<_Iterators...>::__all_bidirectional,
"Attempt to std::prev(it, +n) on a non-bidi iterator");
::cuda::std::advance(__iter, -__n);
return __iter;
}
_CCCL_END_NAMESPACE_STD
# endif // _CCCL_HAS_HOST_STD_LIB()
#endif // _CCCL_DOXYGEN_INVOKED
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___ITERATOR_ZIP_TRANSFORM_ITERATOR_H

73
qwen3_6_scripts/cccl_preload/include/cuda/__memcpy_async/elect_one.h
View File

@@ -1,73 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMCPY_ASYNC_ELECT_ONE_H_
#define _CUDA___MEMCPY_ASYNC_ELECT_ONE_H_
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__memcpy_async/group_traits.h>
#include <cuda/__ptx/instructions/elect_sync.h>
#include <cuda/std/__cccl/prologue.h>
#if _CCCL_CUDA_COMPILATION()
_CCCL_BEGIN_NAMESPACE_CUDA_DEVICE
//! Elects a single leader thread from a one dimensional thread block. For SM90+ will use ptx::elect_sync() etc.,
//! otherwise just selects the thread with ID 0. If the returned value is used as condition for an if statement, the
//! compiler will emit a uniform data path for the branch.
[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE bool __block_elect_one() noexcept
{
_CCCL_ASSERT(blockDim.y == 1 && blockDim.z == 1, "The block must by one dimensional");
NV_IF_ELSE_TARGET(
NV_PROVIDES_SM_90,
(const auto tid = threadIdx.x; //
const auto warp_id = tid / 32;
const auto uniform_warp_id = ::__shfl_sync(~0, warp_id, 0); // broadcast from lane 0
return uniform_warp_id == 0 && ::cuda::ptx::elect_sync(~0); // elect a leader thread among warp 0
),
(return threadIdx.x == 0;));
}
template <typename _Group>
[[nodiscard]] _CCCL_DEVICE_API _CCCL_FORCEINLINE bool __group_elect_one(const _Group& __g) noexcept
{
NV_IF_TARGET(NV_PROVIDES_SM_90,
(
if constexpr (is_thread_block_group_v<_Group>) {
// cooperative groups maps a multidimensional thread id into the thread rank the same way as warps do
const unsigned __tid = __g.thread_rank();
const unsigned __warp_id = __tid / 32;
const unsigned __uniform_warp_id = ::__shfl_sync(~0, __warp_id, 0); // broadcast from lane 0
return __uniform_warp_id == 0 && ::cuda::ptx::elect_sync(~0); // elect a leader thread among warp 0
} else if constexpr (is_warp_group_v<_Group>) { return ::cuda::ptx::elect_sync(~0); }));
return __g.thread_rank() == 0;
}
#endif // _CCCL_CUDA_COMPILATION()
_CCCL_END_NAMESPACE_CUDA_DEVICE
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___MEMCPY_ASYNC_ELECT_ONE_H_

61
qwen3_6_scripts/cccl_preload/include/cuda/__memcpy_async/group_traits.h
View File

@@ -1,61 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMCPY_ASYNC_GROUP_TRAITS_H_
#define _CUDA___MEMCPY_ASYNC_GROUP_TRAITS_H_
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__cccl/prologue.h>
// forward declare cooperative groups types. we cannot include <cooperative_groups.h> since it does not work with NVHPC
namespace cooperative_groups
{
namespace __v1
{
class thread_block;
template <unsigned int Size, typename ParentT>
class thread_block_tile;
} // namespace __v1
using namespace __v1;
} // namespace cooperative_groups
_CCCL_BEGIN_NAMESPACE_CUDA
//! Trait to detect whether a group represents a CUDA thread block, for example: ``cooperative_groups::thread_block``.
template <typename _Group>
inline constexpr bool is_thread_block_group_v = false;
template <>
inline constexpr bool is_thread_block_group_v<::cooperative_groups::thread_block> = true;
//! Trait to detect whether a group represents a CUDA warp, for example:
//! ``cooperative_groups::thread_block_tile<32, ...>``.
template <typename _Group>
inline constexpr bool is_warp_group_v = false;
template <typename _Parent>
inline constexpr bool is_warp_group_v<::cooperative_groups::thread_block_tile<32, _Parent>> = true;
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___MEMCPY_ASYNC_GROUP_TRAITS_H_

78
qwen3_6_scripts/cccl_preload/include/cuda/__memory/align_down.h
View File

@@ -1,78 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMORY_ALIGN_DOWN_H
#define _CUDA___MEMORY_ALIGN_DOWN_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__memory/is_aligned.h>
#include <cuda/__memory/is_valid_alignment.h>
#include <cuda/std/__cstddef/types.h>
#include <cuda/std/__memory/runtime_assume_aligned.h>
#include <cuda/std/__type_traits/is_void.h>
#include <cuda/std/cstdint>
#include <cuda/std/__cccl/prologue.h>
#if _CCCL_HAS_BUILTIN(__builtin_align_down)
# define _CCCL_BUILTIN_ALIGN_DOWN(...) __builtin_align_down(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__builtin_align_down)
// nvcc doesn't support this builtin in device code, clang-cuda crashes
#if (_CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(CLANG)) && _CCCL_DEVICE_COMPILATION()
# undef _CCCL_BUILTIN_ALIGN_DOWN
#endif // (_CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(CLANG)) && _CCCL_DEVICE_COMPILATION()
_CCCL_BEGIN_NAMESPACE_CUDA
template <typename _Tp>
[[nodiscard]] _CCCL_HOST_DEVICE_API _Tp* align_down(_Tp* __ptr, ::cuda::std::size_t __alignment) noexcept
{
using ::cuda::std::uintptr_t;
_CCCL_ASSERT(::cuda::__is_valid_alignment<_Tp>(__alignment), "invalid alignment");
if constexpr (!::cuda::std::is_void_v<_Tp>)
{
_CCCL_ASSERT(::cuda::is_aligned(__ptr, alignof(_Tp)), "__ptr is not aligned for _Tp");
if (__alignment == alignof(_Tp))
{
return __ptr;
}
}
#if defined(_CCCL_BUILTIN_ALIGN_DOWN)
return (_Tp*) _CCCL_BUILTIN_ALIGN_DOWN(__ptr, __alignment);
#else // ^^^ _CCCL_BUILTIN_ALIGN_DOWN ^^^ / vvv !_CCCL_BUILTIN_ALIGN_DOWN vvv
// all code below is translated to a single LOP3.LUT instruction
using _Up = ::cuda::std::remove_cv_t<_Tp>;
const auto __char_ptr = reinterpret_cast<char*>(const_cast<_Up*>(__ptr));
const auto __tmp = static_cast<uintptr_t>(__alignment - 1);
const auto __aligned_ptr = reinterpret_cast<char*>( // NOLINT(performance-no-int-to-ptr)
reinterpret_cast<uintptr_t>(__ptr) & ~__tmp);
// __aligned_ptr and __ptr must be pointers (not values) to apply the optimization
// __ptr - (ptr - aligned_ptr) -> __ptr + (aligned_ptr - ptr)
const auto __diff = static_cast<::cuda::std::size_t>(__aligned_ptr - __char_ptr);
const auto __ret = reinterpret_cast<_Tp*>(__char_ptr + __diff);
return ::cuda::std::__runtime_assume_aligned(__ret, __alignment);
#endif // ^^^ !_CCCL_BUILTIN_ALIGN_DOWN ^^^
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___MEMORY_ALIGN_DOWN_H

78
qwen3_6_scripts/cccl_preload/include/cuda/__memory/align_up.h
View File

@@ -1,78 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMORY_ALIGN_UP_H
#define _CUDA___MEMORY_ALIGN_UP_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__memory/is_aligned.h>
#include <cuda/__memory/is_valid_alignment.h>
#include <cuda/std/__cstddef/types.h>
#include <cuda/std/__memory/runtime_assume_aligned.h>
#include <cuda/std/__type_traits/is_void.h>
#include <cuda/std/__type_traits/remove_cv.h>
#include <cuda/std/cstdint>
#include <cuda/std/__cccl/prologue.h>
#if _CCCL_HAS_BUILTIN(__builtin_align_up)
# define _CCCL_BUILTIN_ALIGN_UP(...) __builtin_align_up(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__builtin_align_up)
// nvcc doesn't support this builtin in device code, clang-cuda crashes
#if (_CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(CLANG)) && _CCCL_DEVICE_COMPILATION()
# undef _CCCL_BUILTIN_ALIGN_UP
#endif // (_CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(CLANG)) && _CCCL_DEVICE_COMPILATION()
_CCCL_BEGIN_NAMESPACE_CUDA
template <typename _Tp>
[[nodiscard]] _CCCL_HOST_DEVICE_API inline _Tp* align_up(_Tp* __ptr, ::cuda::std::size_t __alignment) noexcept
{
using ::cuda::std::uintptr_t;
_CCCL_ASSERT(::cuda::__is_valid_alignment<_Tp>(__alignment), "invalid alignment");
if constexpr (!::cuda::std::is_void_v<_Tp>)
{
_CCCL_ASSERT(::cuda::is_aligned(__ptr, alignof(_Tp)), "__ptr is not aligned for _Tp");
if (__alignment == alignof(_Tp))
{
return __ptr;
}
}
#if defined(_CCCL_BUILTIN_ALIGN_UP)
return (_Tp*) _CCCL_BUILTIN_ALIGN_UP(__ptr, __alignment);
#else // ^^^ _CCCL_BUILTIN_ALIGN_UP ^^^ / vvv !_CCCL_BUILTIN_ALIGN_UP vvv
// all code below is translated to LOP3.LUT + IADD.64 instructions
using _Up = ::cuda::std::remove_cv_t<_Tp>;
const auto __char_ptr = reinterpret_cast<char*>(const_cast<_Up*>(__ptr));
const auto __tmp = static_cast<uintptr_t>(__alignment - 1);
const auto __aligned_ptr = reinterpret_cast<char*>( // NOLINT(performance-no-int-to-ptr)
(reinterpret_cast<uintptr_t>(__ptr) + __tmp) & ~__tmp);
// __aligned_ptr and __ptr must be pointers (not values) to apply the optimization
const auto __diff = static_cast<::cuda::std::size_t>(__aligned_ptr - __char_ptr);
const auto __ret = reinterpret_cast<_Tp*>(__char_ptr + __diff);
return ::cuda::std::__runtime_assume_aligned(__ret, __alignment);
#endif // ^^^ !_CCCL_BUILTIN_ALIGN_UP ^^^
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___MEMORY_ALIGN_UP_H

61
qwen3_6_scripts/cccl_preload/include/cuda/__memory/aligned_size.h
View File

@@ -1,61 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMORY_ALIGNED_SIZE_H
#define _CUDA___MEMORY_ALIGNED_SIZE_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__memory/is_valid_alignment.h>
#include <cuda/std/cstddef>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <::cuda::std::size_t _Alignment>
struct aligned_size_t
{
static_assert(::cuda::__is_valid_alignment(_Alignment), "invalid alignment");
static constexpr ::cuda::std::size_t align = _Alignment;
::cuda::std::size_t value;
_CCCL_API explicit constexpr aligned_size_t(::cuda::std::size_t __s)
: value(__s)
{
_CCCL_ASSERT(value % align == 0,
"aligned_size_t must be constructed with a size that is a multiple of the alignment");
}
_CCCL_API constexpr operator ::cuda::std::size_t() const
{
return value;
}
};
template <class, class = void>
inline constexpr ::cuda::std::size_t __get_size_align_v = 1;
template <class _Tp>
inline constexpr ::cuda::std::size_t __get_size_align_v<_Tp, ::cuda::std::void_t<decltype(_Tp::align)>> = _Tp::align;
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___MEMORY_ALIGNED_SIZE_H

64
qwen3_6_scripts/cccl_preload/include/cuda/__memory/discard_memory.h
View File

@@ -1,64 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMORY_DISCARD_MEMORY_H
#define _CUDA___MEMORY_DISCARD_MEMORY_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__memory/address_space.h>
#include <cuda/__memory/align_down.h>
#include <cuda/__memory/align_up.h>
#include <cuda/std/cstddef>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
_CCCL_HOST_DEVICE_API inline void
discard_memory([[maybe_unused]] volatile void* __ptr, [[maybe_unused]] ::cuda::std::size_t __nbytes) noexcept {
// The discard PTX instruction is only available with PTX ISA 7.4 and later
#if __cccl_ptx_isa >= 740ULL
NV_IF_TARGET(NV_PROVIDES_SM_80, ({
_CCCL_ASSERT(__ptr != nullptr, "null pointer passed to discard_memory");
if (!::cuda::device::is_address_from(__ptr, ::cuda::device::address_space::global))
{
return;
}
constexpr ::cuda::std::size_t __line_size = 128;
// Trim the first block and last block if they're not 128 bytes aligned
const auto __p = static_cast<char*>(const_cast<void*>(__ptr));
const auto __end_p = __p + __nbytes;
const auto __start_aligned = ::cuda::align_up(__p, __line_size);
const auto __end_aligned = ::cuda::align_down(__end_p, __line_size);
for (auto __i = __start_aligned; __i < __end_aligned; __i += __line_size)
{
asm volatile("discard.global.L2 [%0], 128;" ::"l"(__i) :);
}
}))
#endif // __cccl_ptx_isa >= 740ULL
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___MEMORY_DISCARD_MEMORY_H

82
qwen3_6_scripts/cccl_preload/include/cuda/__memory/get_device_address.h
View File

@@ -1,82 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMORY_GET_DEVICE_ADDRESS_H
#define _CUDA___MEMORY_GET_DEVICE_ADDRESS_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_HAS_CTK()
# include <cuda/__device/device_ref.h>
# include <cuda/__runtime/api_wrapper.h>
# include <cuda/__runtime/ensure_current_context.h>
# include <cuda/std/__memory/addressof.h>
# include <nv/target>
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @brief Returns the device address of the passed \c __device_object
//! @param __device_object the object residing in device memory
//! @warning The user must ensure that the current device is properly set to the device the object was allocated on.
//! @return Valid pointer to the device object
template <class _Tp>
[[nodiscard]] _CCCL_API inline _Tp* get_device_address(_Tp& __device_object)
{
NV_IF_ELSE_TARGET(NV_IS_DEVICE, (return ::cuda::std::addressof(__device_object);), ({
void* __device_ptr = nullptr; //
_CCCL_TRY_CUDA_API(::cudaGetSymbolAddress,
"failed to call cudaGetSymbolAddress in cuda::get_device_address",
&__device_ptr,
__device_object);
return static_cast<_Tp*>(__device_ptr);
}))
}
# if !_CCCL_COMPILER(NVRTC)
//! @brief Returns the address of the passed \c __device_object for the passed \c __device.
//!
//! @param __device_object The object residing in device memory.
//! @param __device The device to query the address for.
//!
//! @return Valid pointer to the device object.
//!
//! @throws cuda::cuda_error if the operation fails.
template <class _Tp>
[[nodiscard]] _CCCL_HOST_API inline _Tp* get_device_address(_Tp& __device_object, device_ref __device)
{
__ensure_current_context __ctx{__device};
void* __device_ptr{};
_CCCL_TRY_CUDA_API(::cudaGetSymbolAddress,
"failed to call cudaGetSymbolAddress in cuda::get_device_address",
&__device_ptr,
__device_object);
return static_cast<_Tp*>(__device_ptr);
}
# endif // !_CCCL_COMPILER(NVRTC)
_CCCL_END_NAMESPACE_CUDA
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_CTK()
#endif // _CUDA___MEMORY_GET_DEVICE_ADDRESS_H

60
qwen3_6_scripts/cccl_preload/include/cuda/__memory/is_aligned.h
View File

@@ -1,60 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMORY_IS_ALIGNED_H
#define _CUDA___MEMORY_IS_ALIGNED_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__memory/is_valid_alignment.h>
#include <cuda/std/cstddef>
#include <cuda/std/cstdint>
#include <cuda/std/__cccl/prologue.h>
#if _CCCL_HAS_BUILTIN(__builtin_is_aligned)
# define _CCCL_BUILTIN_IS_ALIGNED(...) __builtin_is_aligned(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__builtin_is_aligned)
// nvcc doesn't support this builtin in device code, clang-cuda crashes
#if _CCCL_CUDA_COMPILER(NVCC) && _CCCL_DEVICE_COMPILATION()
# undef _CCCL_BUILTIN_IS_ALIGNED
#endif // _CCCL_CUDA_COMPILER(NVCC) && _CCCL_DEVICE_COMPILATION()
_CCCL_BEGIN_NAMESPACE_CUDA
[[nodiscard]] _CCCL_API inline bool is_aligned(const void* __ptr, ::cuda::std::size_t __alignment) noexcept
{
_CCCL_ASSERT(::cuda::__is_valid_alignment(__alignment), "invalid alignment");
#if defined(_CCCL_BUILTIN_IS_ALIGNED)
return _CCCL_BUILTIN_IS_ALIGNED(__ptr, __alignment);
#else // ^^^ _CCCL_BUILTIN_IS_ALIGNED ^^^ / vvv !_CCCL_BUILTIN_IS_ALIGNED vvv
return (reinterpret_cast<::cuda::std::uintptr_t>(__ptr) & (__alignment - 1)) == 0;
#endif // ^^^ !_CCCL_BUILTIN_IS_ALIGNED ^^^
}
[[nodiscard]] _CCCL_API inline bool is_aligned(const volatile void* __ptr, ::cuda::std::size_t __alignment) noexcept
{
return ::cuda::is_aligned(const_cast<const void*>(__ptr), __alignment);
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___MEMORY_IS_ALIGN_H

49
qwen3_6_scripts/cccl_preload/include/cuda/__memory/is_valid_alignment.h
View File

@@ -1,49 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMORY_IS_VALID_ALIGNMENT_H
#define _CUDA___MEMORY_IS_VALID_ALIGNMENT_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__cmath/pow2.h>
#include <cuda/std/__cstddef/types.h>
#include <cuda/std/__type_traits/is_void.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <class _Tp = void>
[[nodiscard]] _CCCL_API constexpr bool __is_valid_alignment(::cuda::std::size_t __alignment) noexcept
{
if constexpr (::cuda::std::is_void_v<_Tp>)
{
return __alignment > 0 && ::cuda::is_power_of_two(__alignment);
}
else
{
return __alignment >= alignof(_Tp) && ::cuda::is_power_of_two(__alignment);
}
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___MEMORY_IS_VALID_ALIGNMENT_H

75
qwen3_6_scripts/cccl_preload/include/cuda/__memory/ptr_rebind.h
View File

@@ -1,75 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMORY_PTR_REBIND_H
#define _CUDA___MEMORY_PTR_REBIND_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__memory/assume_aligned.h>
#include <cuda/std/__type_traits/is_same.h>
#include <cuda/std/__type_traits/is_void.h>
#include <cuda/std/cstdint>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <typename _Up, typename _Tp>
[[nodiscard]] _CCCL_HOST_DEVICE_API _Up* ptr_rebind(_Tp* __ptr) noexcept
{
if constexpr (::cuda::std::is_same_v<_Up, _Tp>) // also handle _Tp == _Up == void
{
return __ptr;
}
else if constexpr (::cuda::std::is_void_v<_Up>) // _Tp: non-void, _Up: void
{
_CCCL_ASSERT(reinterpret_cast<::cuda::std::uintptr_t>(__ptr) % alignof(_Tp) == 0, "ptr is not aligned");
return ::cuda::std::assume_aligned<alignof(_Tp)>(reinterpret_cast<_Up*>(__ptr));
}
else
{
constexpr auto __max_alignment = alignof(_Up) > alignof(_Tp) ? alignof(_Up) : alignof(_Tp);
_CCCL_ASSERT(reinterpret_cast<::cuda::std::uintptr_t>(__ptr) % __max_alignment == 0, "ptr is not aligned");
return ::cuda::std::assume_aligned<__max_alignment>(reinterpret_cast<_Up*>(__ptr));
}
}
template <typename _Up, typename _Tp>
[[nodiscard]] _CCCL_HOST_DEVICE_API const _Up* ptr_rebind(const _Tp* __ptr) noexcept
{
return ::cuda::ptr_rebind<const _Up>(const_cast<_Tp*>(__ptr));
}
template <typename _Up, typename _Tp>
[[nodiscard]] _CCCL_HOST_DEVICE_API volatile _Up* ptr_rebind(volatile _Tp* __ptr) noexcept
{
return ::cuda::ptr_rebind<volatile _Up>(const_cast<_Tp*>(__ptr));
}
template <typename _Up, typename _Tp>
[[nodiscard]] _CCCL_HOST_DEVICE_API const volatile _Up* ptr_rebind(const volatile _Tp* __ptr) noexcept
{
return ::cuda::ptr_rebind<const volatile _Up>(const_cast<_Tp*>(__ptr));
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___MEMORY_PTR_REBIND_H

65
qwen3_6_scripts/cccl_preload/include/cuda/__memory/uninitialized_array.h
View File

@@ -1,65 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMORY_UNINITIALIZED_ARRAY_H
#define _CUDA___MEMORY_UNINITIALIZED_ARRAY_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__cstddef/types.h>
#include <cuda/std/__new/launder.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! Aligned storage for _Tp elements (not constructed).
//! Initialize before use with the `data()` method
template <class _Tp, size_t _Size, size_t _Alignment = alignof(_Tp)>
struct __uninitialized_array
{
alignas(_Alignment) unsigned char __data[_Size * sizeof(_Tp)];
[[nodiscard]] _CCCL_API _Tp* data() noexcept
{
return ::cuda::std::launder(reinterpret_cast<_Tp*>(__data));
}
[[nodiscard]] _CCCL_API const _Tp* data() const noexcept
{
return ::cuda::std::launder(reinterpret_cast<const _Tp*>(__data));
}
[[nodiscard]] _CCCL_API _Tp& operator[](const size_t __idx) noexcept
{
_CCCL_ASSERT(__idx < _Size, "out of bounds access in uninitialized_array::operator[]");
return data()[__idx];
}
[[nodiscard]] _CCCL_API const _Tp& operator[](const size_t __idx) const noexcept
{
_CCCL_ASSERT(__idx < _Size, "out of bounds access in uninitialized_array::operator[]");
return data()[__idx];
}
};
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___MEMORY_UNINITIALIZED_ARRAY_H

101
qwen3_6_scripts/cccl_preload/include/cuda/__memory_resource/get_memory_resource.h
View File

@@ -1,101 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMORY_RESOURCE_GET_MEMORY_RESOURCE_H
#define _CUDA___MEMORY_RESOURCE_GET_MEMORY_RESOURCE_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_HAS_CTK()
# include <cuda/__fwd/get_memory_resource.h>
# include <cuda/__memory_resource/properties.h>
# include <cuda/__memory_resource/resource.h>
# include <cuda/__stream/stream_ref.h>
# include <cuda/std/__concepts/equality_comparable.h>
# include <cuda/std/__execution/env.h>
# include <cuda/std/__type_traits/is_same.h>
# include <cuda/std/__type_traits/remove_cvref.h>
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_MR
template <class _Tp>
_CCCL_CONCEPT __has_get_memory_resource_method = _CCCL_REQUIRES_EXPR((_Tp), const _Tp& __t)(__t.get_memory_resource());
template <class _Tp>
_CCCL_CONCEPT __is_synchronous_resource_env = synchronous_resource<_Tp> && !__has_get_memory_resource_method<_Tp>;
template <class _Tp>
_CCCL_CONCEPT __has_member_get_resource = _CCCL_REQUIRES_EXPR((_Tp), const _Tp& __t)(
requires(!__is_synchronous_resource_env<_Tp>),
requires(resource<::cuda::std::remove_cvref_t<decltype(__t.get_memory_resource())>>));
template <class _Env>
_CCCL_CONCEPT __has_query_get_memory_resource = _CCCL_REQUIRES_EXPR((_Env))(
requires(!__is_synchronous_resource_env<_Env>),
requires(!__has_member_get_resource<_Env>),
requires(::cuda::std::execution::__queryable_with<const _Env&, __get_memory_resource_t>));
//! @brief `__get_memory_resource_t` is a customization point object that queries a type `T` for an associated memory
//! resource
struct __get_memory_resource_t
{
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES(__is_synchronous_resource_env<_Tp>)
[[nodiscard]] _CCCL_API constexpr _Tp& operator()(_Tp& __t) const noexcept
{
return __t;
}
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES(__has_member_get_resource<_Tp>)
[[nodiscard]] _CCCL_API constexpr decltype(auto) operator()(const _Tp& __t) const noexcept
{
static_assert(noexcept(__t.get_memory_resource()), "get_memory_resource must be noexcept");
return __t.get_memory_resource();
}
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Env)
_CCCL_REQUIRES(__has_query_get_memory_resource<_Env>)
[[nodiscard]] _CCCL_API constexpr decltype(auto) operator()(const _Env& __env) const noexcept
{
static_assert(noexcept(__env.query(*this)), "get_memory_resource_t query must be noexcept");
static_assert(resource<::cuda::std::remove_cvref_t<decltype(__env.query(*this))>>,
"get_memory_resource_t query must return a cuda::mr::resource");
return __env.query(*this);
}
};
_CCCL_GLOBAL_CONSTANT auto __get_memory_resource = __get_memory_resource_t{};
using get_memory_resource_t = __get_memory_resource_t;
_CCCL_GLOBAL_CONSTANT auto get_memory_resource = get_memory_resource_t{};
_CCCL_END_NAMESPACE_CUDA_MR
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_CTK()
#endif //_CUDA__MEMORY_RESOURCE_GET_MEMORY_RESOURCE_H

202
qwen3_6_scripts/cccl_preload/include/cuda/__memory_resource/get_property.h
View File

@@ -1,202 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMORY_RESOURCE_GET_PROPERTY_H
#define _CUDA___MEMORY_RESOURCE_GET_PROPERTY_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_HAS_CTK()
# include <cuda/__memory_resource/properties.h>
# include <cuda/std/__concepts/same_as.h>
# include <cuda/std/__type_traits/remove_const_ref.h>
# include <cuda/std/__type_traits/void_t.h>
# include <cuda/std/__utility/declval.h>
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @brief The \c has_property concept verifies that a Resource satisfies a given Property
//! @rst
//! For \c has_property we require the following free function to be callable
//!
//! .. code-block:: cpp
//!
//! get_property(const Resource& res, Property prop);
//!
//! @endrst
template <class _Resource, class _Property>
_CCCL_CONCEPT has_property = _CCCL_REQUIRES_EXPR((_Resource, _Property), const _Resource& __res, _Property __prop)(
((void) get_property(__res, __prop)));
template <class _Property>
using __property_value_t = typename _Property::value_type;
//! @brief The \c property_with_value concept verifies that a Property is stateful and signals this through the
//! `value_type` alias
//! @rst
//! .. code-block:: cpp
//!
//! struct stateless_property {};
//! static_assert(!cuda::property_with_value<stateless_property>);
//!
//! struct stateful_property { using value_type = int; };
//! static_assert(!cuda::property_with_value<stateful_property>);
//!
//! @endrst
template <class _Property>
_CCCL_CONCEPT property_with_value = _CCCL_REQUIRES_EXPR((_Property))(typename(__property_value_t<_Property>));
//! @brief The \c has_property_with concept verifies that a Resource satisfies a given stateful Property
//! @rst
//! For \c has_property_with we require the following free function to be callable and its return type to exactly match
//! the ``value_type`` of the Property
//!
//! .. code-block:: cpp
//!
//! struct stateless_property {};
//! constexpr void get_property(const Resource& res, stateless_property) {}
//!
//! // The resource must be stateful
//! static_assert(!cuda::has_property_with<Resource, stateless_property, int>);
//!
//! struct stateful_property { using value_type = int; };
//! constexpr int get_property(const Resource& res, stateful_property) {}
//!
//! // The resource is stateful and has the correct return type
//! static_assert(cuda::has_property_with<Resource, stateful_property, int>);
//!
//! // The resource is stateful but the return type is incorrect
//! static_assert(!cuda::has_property_with<Resource, stateful_property, double>);
//!
//! constexpr double get_property(const OtherResource& res, stateful_property) {}
//!
//! // The resource is stateful but the value_type does not match the `get_property` return type
//! static_assert(!cuda::has_property_with<OtherResource, stateful_property, double>);
//!
//! @endrst
# ifndef _CCCL_DOXYGEN_INVOKED // Doxygen chokes here
template <class _Resource, class _Property, class _Return>
_CCCL_CONCEPT has_property_with = _CCCL_REQUIRES_EXPR((_Resource, _Property, _Return), const _Resource& __res)(
requires(property_with_value<_Property>), _Same_as(_Return) get_property(__res, _Property{}));
template <class _Resource, class _Upstream>
_CCCL_CONCEPT __has_upstream_resource = _CCCL_REQUIRES_EXPR((_Resource, _Upstream), const _Resource& __res)(
requires(::cuda::std::same_as<::cuda::std::__remove_const_ref_t<decltype(__res.upstream_resource())>, _Upstream>));
template <class _Resource, class _Upstream>
_CCCL_CONCEPT __has_get_resource = _CCCL_REQUIRES_EXPR((_Resource, _Upstream), const _Resource& __res)(
requires(::cuda::std::same_as<::cuda::std::__remove_const_ref_t<decltype(__res.get())>, _Upstream>));
# endif // ^^^ _CCCL_DOXYGEN_INVOKED ^^^
template <class _Resource, class _Upstream>
_CCCL_CONCEPT __has_forwarded_resource =
__has_upstream_resource<_Resource, _Upstream> || __has_get_resource<_Resource, _Upstream>;
_CCCL_BEGIN_NAMESPACE_CPO(__forward_property)
template <class _Derived, class _Upstream>
struct __fn
{
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Property)
_CCCL_REQUIRES((!property_with_value<_Property>) _CCCL_AND has_property<_Upstream, _Property>)
_CCCL_API friend constexpr void get_property(const _Derived&, _Property) noexcept {}
_CCCL_EXEC_CHECK_DISABLE
_CCCL_API friend constexpr auto
get_property(const _Derived& __res, ::cuda::mr::dynamic_accessibility_property __prop) noexcept
{
if constexpr (__has_upstream_resource<_Derived, _Upstream>)
{
return get_property(__res.upstream_resource(), __prop);
}
else
{
return get_property(__res.get(), __prop);
}
}
// The indirection is needed, otherwise the compiler might believe that _Derived is an incomplete type
_CCCL_EXEC_CHECK_DISABLE
_CCCL_TEMPLATE(class _Property, class _Derived2 = _Derived)
_CCCL_REQUIRES(property_with_value<_Property> _CCCL_AND has_property<_Upstream, _Property> _CCCL_AND
__has_forwarded_resource<_Derived2, _Upstream>)
_CCCL_API friend constexpr __property_value_t<_Property> get_property(const _Derived& __res, _Property __prop)
{
if constexpr (__has_upstream_resource<_Derived, _Upstream>)
{
return get_property(__res.upstream_resource(), __prop);
}
else
{
return get_property(__res.get(), __prop);
}
}
};
_CCCL_END_NAMESPACE_CPO
//! @brief The \c forward_property CRTP template allows Derived to forward all properties of Upstream
//! @rst
//! .. code-block:: cpp
//!
//! class UpstreamWithProperties;
//!
//! class DerivedClass : cuda::forward_properties<DerivedClass, UpstreamWithProperties> {
//! // This method is needed to forward stateful properties
//! UpstreamWithProperties& upstream_resource() const { ... }
//! };
//!
//! .. note::
//!
//! In order to forward stateful properties, a type needs to implement either:
//! - an `upstream_resource()` method returning the upstream resource, or
//! - a `get()` method returning the upstream resource.
//!
//! @endrst
template <class _Derived, class _Upstream>
using forward_property = __forward_property::__fn<_Derived, _Upstream>;
_CCCL_END_NAMESPACE_CUDA
_CCCL_BEGIN_NAMESPACE_CUDA_MR
template <class _Tp>
inline constexpr bool __disable_default_dynamic_accessibility_property = false;
//! Default implementation: infer from has_property<Resource, host_accessible> and
//! has_property<Resource, device_accessible>. Resources can override by providing
//! their own get_property(..., dynamic_accessibility_property).
//! Excluded for type-erased wrappers (any_resource, resource_ref, etc.) so that
//! get_property dispatches via their interface to the stored concrete resource.
_CCCL_TEMPLATE(class _Resource)
_CCCL_REQUIRES((!__disable_default_dynamic_accessibility_property<_Resource>) )
_CCCL_API constexpr __memory_accessibility
get_property([[maybe_unused]] const _Resource& __res, dynamic_accessibility_property) noexcept
{
return __memory_accessibility_from_static_properties<::cuda::has_property<_Resource, host_accessible>,
::cuda::has_property<_Resource, device_accessible>>();
}
_CCCL_END_NAMESPACE_CUDA_MR
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_CTK()
#endif //_CUDA___MEMORY_RESOURCE_GET_PROPERTY_H

150
qwen3_6_scripts/cccl_preload/include/cuda/__memory_resource/properties.h
View File

@@ -1,150 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMORY_RESOURCE_PROPERTIES_H
#define _CUDA___MEMORY_RESOURCE_PROPERTIES_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_HAS_CTK()
# include <cuda/std/__type_traits/decay.h>
# include <cuda/std/__type_traits/type_set.h>
# include <cuda/std/cstddef>
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_MR
//! @brief The default alignment by a cudaMalloc{...} call
inline constexpr size_t default_cuda_malloc_alignment = 256;
//! @brief The default alignment by a cudaMallocHost{...} call
inline constexpr size_t default_cuda_malloc_host_alignment = alignof(::cuda::std::max_align_t);
//! @brief The device_accessible property signals that the allocated memory is device accessible
struct device_accessible
{};
//! @brief The device_accessible property signals that the allocated memory is host accessible
struct host_accessible
{};
//! @brief determines whether a set of properties signals host accessible memory.
template <class... _Properties>
inline constexpr bool __is_host_accessible =
::cuda::std::__type_set_contains_v<::cuda::std::__make_type_set<_Properties...>, host_accessible>;
//! @brief determines whether a set of properties signals device accessible memory.
template <class... _Properties>
inline constexpr bool __is_device_accessible =
::cuda::std::__type_set_contains_v<::cuda::std::__make_type_set<_Properties...>, device_accessible>;
//! @brief determines whether a set of properties signals host device accessible memory.
template <class... _Properties>
inline constexpr bool __is_host_device_accessible =
::cuda::std::__type_set_contains_v<::cuda::std::__make_type_set<_Properties...>, host_accessible, device_accessible>;
//! @brief verifies that a set of properties contains at least one execution space property
template <class... _Properties>
inline constexpr bool __contains_execution_space_property =
__is_host_accessible<_Properties...> || __is_device_accessible<_Properties...>;
//! @brief A type representing a list of memory resource properties
//! @tparam _Properties The properties to be included in the list
//! It has a member template `rebind` that allows constructing a type by combining
//! a template and type arguments with the properties from this list. The properties
//! are appended after the type arguments in the resulting type.
template <class... _Properties>
struct properties_list
{
//! @brief A type alias for a type template instantiated with the properties
//! from this list appended to the type arguments.
template <template <class...> class _Fn, class... _ExtraArgs>
using rebind = _Fn<_ExtraArgs..., _Properties...>;
template <class _QueryProperty>
_CCCL_HOST_API static constexpr bool has_property([[maybe_unused]] _QueryProperty)
{
return ::cuda::std::__type_set_contains_v<::cuda::std::__make_type_set<_Properties...>, _QueryProperty>;
}
};
template <class _Tp>
inline constexpr bool __is_queries_list = false;
template <class... _Tp>
inline constexpr bool __is_queries_list<properties_list<_Tp...>> = true;
template <typename _Tp>
_CCCL_CONCEPT __has_default_queries =
_CCCL_REQUIRES_EXPR((_Tp))(requires(__is_queries_list<typename ::cuda::std::decay_t<_Tp>::default_queries>));
template <typename _Resource, bool _HasDefaultQueries = __has_default_queries<_Resource>>
struct __copy_default_queries;
template <typename _Resource>
struct __copy_default_queries<_Resource, true>
{
using default_queries = typename _Resource::default_queries;
};
template <typename _Resource>
struct __copy_default_queries<_Resource, false>
{};
enum class __memory_accessibility
{
__unknown,
__host,
__device,
__host_device,
};
//! @brief The dynamic_accessibility_property reports the resource's memory accessibility at runtime.
//! Compared to the static properties, it can be used to query the memory accessibility of a resource that is not known
//! at compile time.
struct dynamic_accessibility_property
{
using value_type = __memory_accessibility;
};
template <bool _HostAccessible, bool _DeviceAccessible>
_CCCL_API constexpr __memory_accessibility __memory_accessibility_from_static_properties() noexcept
{
return _HostAccessible && _DeviceAccessible ? __memory_accessibility ::__host_device
: _DeviceAccessible ? __memory_accessibility ::__device
: _HostAccessible ? __memory_accessibility ::__host
: __memory_accessibility ::__unknown;
}
template <class... _Properties>
struct __memory_accessibility_from_properties
{
static constexpr __memory_accessibility value =
__memory_accessibility_from_static_properties<::cuda::mr::__is_host_accessible<_Properties...>,
::cuda::mr::__is_device_accessible<_Properties...>>();
};
_CCCL_END_NAMESPACE_CUDA_MR
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_CTK()
#endif //_CUDA___MEMORY_RESOURCE_PROPERTIES_H

138
qwen3_6_scripts/cccl_preload/include/cuda/__memory_resource/resource.h
View File

@@ -1,138 +0,0 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___MEMORY_RESOURCE_RESOURCE_H
#define _CUDA___MEMORY_RESOURCE_RESOURCE_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_HAS_CTK()
# include <cuda/__memory_resource/get_property.h>
# include <cuda/__stream/stream_ref.h>
# include <cuda/__utility/__basic_any/semiregular.h>
# include <cuda/std/__concepts/concept_macros.h>
# include <cuda/std/__concepts/convertible_to.h>
# include <cuda/std/__concepts/equality_comparable.h>
# include <cuda/std/__concepts/same_as.h>
# include <cuda/std/__type_traits/decay.h>
# include <cuda/std/__type_traits/fold.h>
# include <cuda/std/__type_traits/is_same.h>
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_MR
//! @brief The \c synchronous_resource concept verifies that a type Resource satisfies the basic requirements of a
//! memory resource
//! @rst
//! We require that a resource supports the following interface
//!
//! - ``allocate_sync(size_t bytes, size_t alignment)``
//! - ``deallocate_sync(void* ptr, size_t bytes, size_t alignment)``
//! - ``T() == T()``
//! - ``T() != T()``
//!
//! @endrst
//! @tparam _Resource The type that should implement the synchronous resource concept
template <class _Resource>
_CCCL_CONCEPT synchronous_resource =
_CCCL_REQUIRES_EXPR((_Resource), _Resource& __res, void* __ptr, size_t __bytes, size_t __alignment)(
_Same_as(void*) __res.allocate_sync(__bytes, __alignment), //
_Same_as(void) __res.deallocate_sync(__ptr, __bytes, __alignment),
requires(::cuda::std::equality_comparable<_Resource>));
//! @brief The \c resource concept verifies that a type Resource satisfies the basic requirements of a
//! memory resource and additionally supports stream ordered allocations
//! @rst
//! We require that an resource supports the following interface
//!
//! - ``allocate_sync(size_t bytes, size_t alignment)``
//! - ``deallocate_sync(void* ptr, size_t bytes, size_t alignment)``
//! - ``T() == T()``
//! - ``T() != T()``
//!
//! - ``allocate(cuda::stream_ref stream, size_t bytes, size_t alignment)``
//! - ``deallocate( cuda::stream_ref stream, void* ptr, size_t bytes, size_t alignment)``
//!
//! @endrst
//! @tparam _Resource The type that should implement the resource concept
template <class _Resource>
_CCCL_CONCEPT resource = _CCCL_REQUIRES_EXPR(
(_Resource), _Resource& __res, void* __ptr, size_t __bytes, size_t __alignment, ::cuda::stream_ref __stream)(
_Same_as(void*) __res.allocate(__stream, __bytes, __alignment),
_Same_as(void) __res.deallocate(__stream, __ptr, __bytes, __alignment),
requires(synchronous_resource<_Resource>));
//! @brief The \c resource_with concept verifies that a type Resource satisfies the `synchronous_resource` concept and
//! also satisfies all the provided Properties
//! @tparam _Resource
//! @tparam _Properties
// We cannot use fold expressions here due to a nvcc bug
template <class _Resource, class... _Properties>
_CCCL_CONCEPT synchronous_resource_with = _CCCL_REQUIRES_EXPR((_Resource, variadic _Properties))(
requires(synchronous_resource<_Resource>),
requires(::cuda::std::__fold_and_v<has_property<_Resource, _Properties>...>));
//! @brief The \c resource_with concept verifies that a type Resource satisfies the `resource`
//! concept and also satisfies all the provided Properties
//! @tparam _Resource
//! @tparam _Properties
// We cannot use fold expressions here due to a nvcc bug
template <class _Resource, class... _Properties>
_CCCL_CONCEPT resource_with = _CCCL_REQUIRES_EXPR((_Resource, variadic _Properties))(
requires(resource<_Resource>), requires(::cuda::std::__fold_and_v<has_property<_Resource, _Properties>...>));
template <bool _Convertible>
struct __different_resource__
{
template <class _OtherResource>
static constexpr bool __value(_OtherResource*) noexcept
{
return synchronous_resource<_OtherResource>;
}
};
template <>
struct __different_resource__<true>
{
static constexpr bool __value(void*) noexcept
{
return false;
}
};
template <class _Resource, class _OtherResource>
_CCCL_CONCEPT __different_resource =
__different_resource__<::cuda::std::convertible_to<_OtherResource const&, _Resource const&>>::__value(
static_cast<_OtherResource*>(nullptr));
template <class _Resource, class _OtherResource>
_CCCL_CONCEPT __non_polymorphic_resources = _CCCL_REQUIRES_EXPR((_Resource, _OtherResource))(
requires(::cuda::mr::synchronous_resource<_Resource>),
requires(::cuda::mr::synchronous_resource<_OtherResource>),
requires(::cuda::__non_polymorphic<_Resource>),
requires(::cuda::__non_polymorphic<_OtherResource>));
_CCCL_END_NAMESPACE_CUDA_MR
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_CTK()
#endif //_CUDA___MEMORY_RESOURCE_RESOURCE_H

44
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/bfind.h
View File

@@ -1,44 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_PTX_BFIND_H_
#define _CUDA_PTX_BFIND_H_
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__ptx/ptx_dot_variants.h>
#include <cuda/__ptx/ptx_helper_functions.h>
#include <cuda/std/__type_traits/is_integral.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__type_traits/is_unsigned.h>
#include <cuda/std/cstdint>
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
#include <cuda/__ptx/instructions/generated/bfind.h>
_CCCL_END_NAMESPACE_CUDA_PTX
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA_PTX_BFIND_H_

41
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/bmsk.h
View File

@@ -1,41 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_PTX_BMSK_H_
#define _CUDA_PTX_BMSK_H_
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__ptx/ptx_dot_variants.h>
#include <cuda/__ptx/ptx_helper_functions.h>
#include <cuda/std/cstdint>
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
#include <cuda/__ptx/instructions/generated/bmsk.h>
_CCCL_END_NAMESPACE_CUDA_PTX
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA_PTX_BMSK_H_

41
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/clusterlaunchcontrol.h
View File

@@ -1,41 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_PTX_CLUSTERLAUNCHCONTROL_H_
#define _CUDA_PTX_CLUSTERLAUNCHCONTROL_H_
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__ptx/ptx_dot_variants.h>
#include <cuda/__ptx/ptx_helper_functions.h>
#include <cuda/std/cstdint>
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
#include <cuda/__ptx/instructions/generated/clusterlaunchcontrol.h>
_CCCL_END_NAMESPACE_CUDA_PTX
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA_PTX_CLUSTERLAUNCHCONTROL_H_

44
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/cp_async_bulk.h
View File

@@ -1,44 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_PTX_CP_ASYNC_BULK_H_
#define _CUDA_PTX_CP_ASYNC_BULK_H_
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__ptx/ptx_dot_variants.h>
#include <cuda/__ptx/ptx_helper_functions.h>
#include <cuda/std/cstdint>
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
// 9.7.8.24.6. Data Movement and Conversion Instructions: cp.async.bulk
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk
#include <cuda/__ptx/instructions/generated/cp_async_bulk.h>
#include <cuda/__ptx/instructions/generated/cp_async_bulk_multicast.h>
_CCCL_END_NAMESPACE_CUDA_PTX
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA_PTX_CP_ASYNC_BULK_H_

43
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/cp_async_bulk_commit_group.h
View File

@@ -1,43 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_PTX_CP_ASYNC_BULK_COMMIT_GROUP_H_
#define _CUDA_PTX_CP_ASYNC_BULK_COMMIT_GROUP_H_
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__ptx/ptx_dot_variants.h>
#include <cuda/__ptx/ptx_helper_functions.h>
#include <cuda/std/cstdint>
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
// 9.7.8.24.12. Data Movement and Conversion Instructions: cp.async.bulk.commit_group
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-commit-group
#include <cuda/__ptx/instructions/generated/cp_async_bulk_commit_group.h>
_CCCL_END_NAMESPACE_CUDA_PTX
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA_PTX_CP_ASYNC_BULK_COMMIT_GROUP_H_

43
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/cp_async_bulk_wait_group.h
View File

@@ -1,43 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_PTX_CP_ASYNC_BULK_WAIT_GROUP_H_
#define _CUDA_PTX_CP_ASYNC_BULK_WAIT_GROUP_H_
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__ptx/ptx_dot_variants.h>
#include <cuda/__ptx/ptx_helper_functions.h>
#include <cuda/std/cstdint>
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
// 9.7.8.24.13. Data Movement and Conversion Instructions: cp.async.bulk.wait_group
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-wait-group
#include <cuda/__ptx/instructions/generated/cp_async_bulk_wait_group.h>
_CCCL_END_NAMESPACE_CUDA_PTX
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA_PTX_CP_ASYNC_BULK_WAIT_GROUP_H_

41
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/elect_sync.h
View File

@@ -1,41 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_PTX_ELECT_SYNC_H_
#define _CUDA_PTX_ELECT_SYNC_H_
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__ptx/ptx_dot_variants.h>
#include <cuda/__ptx/ptx_helper_functions.h>
#include <cuda/std/cstdint>
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
#include <cuda/__ptx/instructions/generated/elect_sync.h>
_CCCL_END_NAMESPACE_CUDA_PTX
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA_PTX_ELECT_SYNC_H_

49
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/fence.h
View File

@@ -1,49 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_PTX_FENCE_H_
#define _CUDA_PTX_FENCE_H_
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__ptx/ptx_dot_variants.h>
#include <cuda/__ptx/ptx_helper_functions.h>
#include <cuda/std/cstdint>
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
// 9.7.12.4. Parallel Synchronization and Communication Instructions: membar/fence
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar-fence
#include <cuda/__ptx/instructions/generated/fence.h>
#include <cuda/__ptx/instructions/generated/fence_mbarrier_init.h>
#include <cuda/__ptx/instructions/generated/fence_proxy_alias.h>
#include <cuda/__ptx/instructions/generated/fence_proxy_async.h>
#include <cuda/__ptx/instructions/generated/fence_proxy_async_generic_sync_restrict.h>
#include <cuda/__ptx/instructions/generated/fence_proxy_tensormap_generic.h>
#include <cuda/__ptx/instructions/generated/fence_sync_restrict.h>
_CCCL_END_NAMESPACE_CUDA_PTX
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA_PTX_FENCE_H_

154
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/bfind.h
View File

@@ -1,154 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_BFIND_H_
#define _CUDA_PTX_GENERATED_BFIND_H_
/*
// bfind.u32 dest, a_reg; // PTX ISA 20, SM_50
template <typename U32, enable_if_t<sizeof(U32) == 4 && is_integral_v<U32> && is_unsigned_v<U32>, bool> = true>
__device__ static inline uint32_t bfind(
U32 a_reg);
*/
#if __cccl_ptx_isa >= 200
template <
typename _U32,
::cuda::std::enable_if_t<sizeof(_U32) == 4 && ::cuda::std::is_integral_v<_U32>&& ::cuda::std::is_unsigned_v<_U32>,
bool> = true>
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind(_U32 __a_reg)
{
::cuda::std::uint32_t __dest;
asm("bfind.u32 %0, %1;" : "=r"(__dest) : "r"(*reinterpret_cast<const ::cuda::std::uint32_t*>(&__a_reg)) :);
return __dest;
}
#endif // __cccl_ptx_isa >= 200
/*
// bfind.shiftamt.u32 dest, a_reg; // PTX ISA 20, SM_50
template <typename U32, enable_if_t<sizeof(U32) == 4 && is_integral_v<U32> && is_unsigned_v<U32>, bool> = true>
__device__ static inline uint32_t bfind_shiftamt(
U32 a_reg);
*/
#if __cccl_ptx_isa >= 200
template <
typename _U32,
::cuda::std::enable_if_t<sizeof(_U32) == 4 && ::cuda::std::is_integral_v<_U32>&& ::cuda::std::is_unsigned_v<_U32>,
bool> = true>
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind_shiftamt(_U32 __a_reg)
{
::cuda::std::uint32_t __dest;
asm("bfind.shiftamt.u32 %0, %1;" : "=r"(__dest) : "r"(*reinterpret_cast<const ::cuda::std::uint32_t*>(&__a_reg)) :);
return __dest;
}
#endif // __cccl_ptx_isa >= 200
/*
// bfind.u64 dest, a_reg; // PTX ISA 20, SM_50
template <typename U64, enable_if_t<sizeof(U64) == 8 && is_integral_v<U64> && is_unsigned_v<U64>, bool> = true>
__device__ static inline uint32_t bfind(
U64 a_reg);
*/
#if __cccl_ptx_isa >= 200
template <
typename _U64,
::cuda::std::enable_if_t<sizeof(_U64) == 8 && ::cuda::std::is_integral_v<_U64>&& ::cuda::std::is_unsigned_v<_U64>,
bool> = true>
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind(_U64 __a_reg)
{
::cuda::std::uint32_t __dest;
asm("bfind.u64 %0, %1;" : "=r"(__dest) : "l"(*reinterpret_cast<const ::cuda::std::uint64_t*>(&__a_reg)) :);
return __dest;
}
#endif // __cccl_ptx_isa >= 200
/*
// bfind.shiftamt.u64 dest, a_reg; // PTX ISA 20, SM_50
template <typename U64, enable_if_t<sizeof(U64) == 8 && is_integral_v<U64> && is_unsigned_v<U64>, bool> = true>
__device__ static inline uint32_t bfind_shiftamt(
U64 a_reg);
*/
#if __cccl_ptx_isa >= 200
template <
typename _U64,
::cuda::std::enable_if_t<sizeof(_U64) == 8 && ::cuda::std::is_integral_v<_U64>&& ::cuda::std::is_unsigned_v<_U64>,
bool> = true>
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind_shiftamt(_U64 __a_reg)
{
::cuda::std::uint32_t __dest;
asm("bfind.shiftamt.u64 %0, %1;" : "=r"(__dest) : "l"(*reinterpret_cast<const ::cuda::std::uint64_t*>(&__a_reg)) :);
return __dest;
}
#endif // __cccl_ptx_isa >= 200
/*
// bfind.s32 dest, a_reg; // PTX ISA 20, SM_50
template <typename S32, enable_if_t<sizeof(S32) == 4 && is_integral_v<S32> && is_signed_v<S32>, bool> = true>
__device__ static inline uint32_t bfind(
S32 a_reg);
*/
#if __cccl_ptx_isa >= 200
template <typename _S32,
::cuda::std::enable_if_t<sizeof(_S32) == 4 && ::cuda::std::is_integral_v<_S32>&& ::cuda::std::is_signed_v<_S32>,
bool> = true>
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind(_S32 __a_reg)
{
::cuda::std::uint32_t __dest;
asm("bfind.s32 %0, %1;" : "=r"(__dest) : "r"(*reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)) :);
return __dest;
}
#endif // __cccl_ptx_isa >= 200
/*
// bfind.shiftamt.s32 dest, a_reg; // PTX ISA 20, SM_50
template <typename S32, enable_if_t<sizeof(S32) == 4 && is_integral_v<S32> && is_signed_v<S32>, bool> = true>
__device__ static inline uint32_t bfind_shiftamt(
S32 a_reg);
*/
#if __cccl_ptx_isa >= 200
template <typename _S32,
::cuda::std::enable_if_t<sizeof(_S32) == 4 && ::cuda::std::is_integral_v<_S32>&& ::cuda::std::is_signed_v<_S32>,
bool> = true>
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind_shiftamt(_S32 __a_reg)
{
::cuda::std::uint32_t __dest;
asm("bfind.shiftamt.s32 %0, %1;" : "=r"(__dest) : "r"(*reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)) :);
return __dest;
}
#endif // __cccl_ptx_isa >= 200
/*
// bfind.s64 dest, a_reg; // PTX ISA 20, SM_50
template <typename S64, enable_if_t<sizeof(S64) == 8 && is_integral_v<S64> && is_signed_v<S64>, bool> = true>
__device__ static inline uint32_t bfind(
S64 a_reg);
*/
#if __cccl_ptx_isa >= 200
template <typename _S64,
::cuda::std::enable_if_t<sizeof(_S64) == 8 && ::cuda::std::is_integral_v<_S64>&& ::cuda::std::is_signed_v<_S64>,
bool> = true>
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind(_S64 __a_reg)
{
::cuda::std::uint32_t __dest;
asm("bfind.s64 %0, %1;" : "=r"(__dest) : "l"(*reinterpret_cast<const ::cuda::std::int64_t*>(&__a_reg)) :);
return __dest;
}
#endif // __cccl_ptx_isa >= 200
/*
// bfind.shiftamt.s64 dest, a_reg; // PTX ISA 20, SM_50
template <typename S64, enable_if_t<sizeof(S64) == 8 && is_integral_v<S64> && is_signed_v<S64>, bool> = true>
__device__ static inline uint32_t bfind_shiftamt(
S64 a_reg);
*/
#if __cccl_ptx_isa >= 200
template <typename _S64,
::cuda::std::enable_if_t<sizeof(_S64) == 8 && ::cuda::std::is_integral_v<_S64>&& ::cuda::std::is_signed_v<_S64>,
bool> = true>
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind_shiftamt(_S64 __a_reg)
{
::cuda::std::uint32_t __dest;
asm("bfind.shiftamt.s64 %0, %1;" : "=r"(__dest) : "l"(*reinterpret_cast<const ::cuda::std::int64_t*>(&__a_reg)) :);
return __dest;
}
#endif // __cccl_ptx_isa >= 200
#endif // _CUDA_PTX_GENERATED_BFIND_H_

54
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/bmsk.h
View File

@@ -1,54 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_BMSK_H_
#define _CUDA_PTX_GENERATED_BMSK_H_
/*
// bmsk.clamp.b32 dest, a_reg, b_reg; // PTX ISA 76, SM_70
template <typename = void>
__device__ static inline uint32_t bmsk_clamp(
uint32_t a_reg,
uint32_t b_reg);
*/
#if __cccl_ptx_isa >= 760
extern "C" _CCCL_DEVICE void __cuda_ptx_bmsk_clamp_is_not_supported_before_SM_70__();
template <typename = void>
_CCCL_DEVICE static inline ::cuda::std::uint32_t bmsk_clamp(::cuda::std::uint32_t __a_reg, ::cuda::std::uint32_t __b_reg)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 700
::cuda::std::uint32_t __dest;
asm("bmsk.clamp.b32 %0, %1, %2;" : "=r"(__dest) : "r"(__a_reg), "r"(__b_reg) :);
return __dest;
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_bmsk_clamp_is_not_supported_before_SM_70__();
return 0;
# endif
}
#endif // __cccl_ptx_isa >= 760
/*
// bmsk.wrap.b32 dest, a_reg, b_reg; // PTX ISA 76, SM_70
template <typename = void>
__device__ static inline uint32_t bmsk_wrap(
uint32_t a_reg,
uint32_t b_reg);
*/
#if __cccl_ptx_isa >= 760
extern "C" _CCCL_DEVICE void __cuda_ptx_bmsk_wrap_is_not_supported_before_SM_70__();
template <typename = void>
_CCCL_DEVICE static inline ::cuda::std::uint32_t bmsk_wrap(::cuda::std::uint32_t __a_reg, ::cuda::std::uint32_t __b_reg)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 700
::cuda::std::uint32_t __dest;
asm("bmsk.wrap.b32 %0, %1, %2;" : "=r"(__dest) : "r"(__a_reg), "r"(__b_reg) :);
return __dest;
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_bmsk_wrap_is_not_supported_before_SM_70__();
return 0;
# endif
}
#endif // __cccl_ptx_isa >= 760
#endif // _CUDA_PTX_GENERATED_BMSK_H_

240
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/clusterlaunchcontrol.h
View File

@@ -1,240 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_CLUSTERLAUNCHCONTROL_H_
#define _CUDA_PTX_GENERATED_CLUSTERLAUNCHCONTROL_H_
/*
// clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [addr], [smem_bar]; // PTX ISA
86, SM_100 template <typename = void>
__device__ static inline void clusterlaunchcontrol_try_cancel(
void* addr,
uint64_t* smem_bar);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_clusterlaunchcontrol_try_cancel_is_not_supported_before_SM_100__();
template <typename = void>
_CCCL_DEVICE static inline void clusterlaunchcontrol_try_cancel(void* __addr, ::cuda::std::uint64_t* __smem_bar)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
asm("clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [%0], [%1];"
:
: "r"(__as_ptr_smem(__addr)), "r"(__as_ptr_smem(__smem_bar))
: "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_clusterlaunchcontrol_try_cancel_is_not_supported_before_SM_100__();
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.multicast::cluster::all.b128 [addr],
[smem_bar]; // PTX ISA 86, SM_100a, SM_110a template <typename = void>
__device__ static inline void clusterlaunchcontrol_try_cancel_multicast(
void* addr,
uint64_t* smem_bar);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_clusterlaunchcontrol_try_cancel_multicast_is_only_supported_on_SM_100a_110a__();
template <typename = void>
_CCCL_DEVICE static inline void
clusterlaunchcontrol_try_cancel_multicast(void* __addr, ::cuda::std::uint64_t* __smem_bar)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
asm("clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.multicast::cluster::all.b128 "
"[%0], [%1];"
:
: "r"(__as_ptr_smem(__addr)), "r"(__as_ptr_smem(__smem_bar))
: "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_clusterlaunchcontrol_try_cancel_multicast_is_only_supported_on_SM_100a_110a__();
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 pred_is_canceled, try_cancel_response; // PTX ISA 86, SM_100
template <typename B128, enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline bool clusterlaunchcontrol_query_cancel_is_canceled(
B128 try_cancel_response);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_clusterlaunchcontrol_query_cancel_is_canceled_is_not_supported_before_SM_100__();
template <typename _B128, ::cuda::std::enable_if_t<sizeof(_B128) == 16, bool> = true>
_CCCL_DEVICE static inline bool clusterlaunchcontrol_query_cancel_is_canceled(_B128 __try_cancel_response)
{
static_assert(sizeof(_B128) == 16);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
::cuda::std::uint32_t __pred_is_canceled;
asm("{\n\t .reg .b128 B128_try_cancel_response; \n\t"
"mov.b128 B128_try_cancel_response, {%1, %2}; \n"
"{\n\t .reg .pred P_OUT; \n\t"
"clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 P_OUT, B128_try_cancel_response;\n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}\n\t"
"}"
: "=r"(__pred_is_canceled)
: "l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).x),
"l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).y)
:);
return static_cast<bool>(__pred_is_canceled);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_clusterlaunchcontrol_query_cancel_is_canceled_is_not_supported_before_SM_100__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// clusterlaunchcontrol.query_cancel.get_first_ctaid::x.b32.b128 ret_dim, try_cancel_response; // PTX ISA 86, SM_100
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, enable_if_t<sizeof(B128) == 16, bool>
= true>
__device__ static inline B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_x(
B128 try_cancel_response);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_x_is_not_supported_before_SM_100__();
template <typename _B32,
::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true,
typename _B128,
::cuda::std::enable_if_t<sizeof(_B128) == 16, bool> = true>
_CCCL_DEVICE static inline _B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_x(_B128 __try_cancel_response)
{
static_assert(sizeof(_B32) == 4);
static_assert(sizeof(_B128) == 16);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
::cuda::std::uint32_t __ret_dim;
asm("{\n\t .reg .b128 B128_try_cancel_response; \n\t"
"mov.b128 B128_try_cancel_response, {%1, %2}; \n"
"clusterlaunchcontrol.query_cancel.get_first_ctaid::x.b32.b128 %0, B128_try_cancel_response;\n\t"
"}"
: "=r"(__ret_dim)
: "l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).x),
"l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).y)
:);
return *reinterpret_cast<_B32*>(&__ret_dim);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_x_is_not_supported_before_SM_100__();
::cuda::std::uint32_t __err_out_var = 0;
return *reinterpret_cast<_B32*>(&__err_out_var);
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// clusterlaunchcontrol.query_cancel.get_first_ctaid::y.b32.b128 ret_dim, try_cancel_response; // PTX ISA 86, SM_100
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, enable_if_t<sizeof(B128) == 16, bool>
= true>
__device__ static inline B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_y(
B128 try_cancel_response);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_y_is_not_supported_before_SM_100__();
template <typename _B32,
::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true,
typename _B128,
::cuda::std::enable_if_t<sizeof(_B128) == 16, bool> = true>
_CCCL_DEVICE static inline _B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_y(_B128 __try_cancel_response)
{
static_assert(sizeof(_B32) == 4);
static_assert(sizeof(_B128) == 16);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
::cuda::std::uint32_t __ret_dim;
asm("{\n\t .reg .b128 B128_try_cancel_response; \n\t"
"mov.b128 B128_try_cancel_response, {%1, %2}; \n"
"clusterlaunchcontrol.query_cancel.get_first_ctaid::y.b32.b128 %0, B128_try_cancel_response;\n\t"
"}"
: "=r"(__ret_dim)
: "l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).x),
"l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).y)
:);
return *reinterpret_cast<_B32*>(&__ret_dim);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_y_is_not_supported_before_SM_100__();
::cuda::std::uint32_t __err_out_var = 0;
return *reinterpret_cast<_B32*>(&__err_out_var);
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// clusterlaunchcontrol.query_cancel.get_first_ctaid::z.b32.b128 ret_dim, try_cancel_response; // PTX ISA 86, SM_100
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, enable_if_t<sizeof(B128) == 16, bool>
= true>
__device__ static inline B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_z(
B128 try_cancel_response);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_z_is_not_supported_before_SM_100__();
template <typename _B32,
::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true,
typename _B128,
::cuda::std::enable_if_t<sizeof(_B128) == 16, bool> = true>
_CCCL_DEVICE static inline _B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_z(_B128 __try_cancel_response)
{
static_assert(sizeof(_B32) == 4);
static_assert(sizeof(_B128) == 16);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
::cuda::std::uint32_t __ret_dim;
asm("{\n\t .reg .b128 B128_try_cancel_response; \n\t"
"mov.b128 B128_try_cancel_response, {%1, %2}; \n"
"clusterlaunchcontrol.query_cancel.get_first_ctaid::z.b32.b128 %0, B128_try_cancel_response;\n\t"
"}"
: "=r"(__ret_dim)
: "l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).x),
"l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).y)
:);
return *reinterpret_cast<_B32*>(&__ret_dim);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_z_is_not_supported_before_SM_100__();
::cuda::std::uint32_t __err_out_var = 0;
return *reinterpret_cast<_B32*>(&__err_out_var);
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 block_dim, try_cancel_response; // PTX ISA 86, SM_100
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, enable_if_t<sizeof(B128) == 16, bool>
= true>
__device__ static inline void clusterlaunchcontrol_query_cancel_get_first_ctaid(
B32 (&block_dim)[4],
B128 try_cancel_response);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_is_not_supported_before_SM_100__();
template <typename _B32,
::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true,
typename _B128,
::cuda::std::enable_if_t<sizeof(_B128) == 16, bool> = true>
_CCCL_DEVICE static inline void
clusterlaunchcontrol_query_cancel_get_first_ctaid(_B32 (&__block_dim)[4], _B128 __try_cancel_response)
{
static_assert(sizeof(_B32) == 4);
static_assert(sizeof(_B128) == 16);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
asm("{\n\t .reg .b128 B128_try_cancel_response; \n\t"
"mov.b128 B128_try_cancel_response, {%4, %5}; \n"
"clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {%0, %1, %2, %3}, B128_try_cancel_response;\n\t"
"}"
: "=r"(__block_dim[0]), "=r"(__block_dim[1]), "=r"(__block_dim[2]), "=r"(__block_dim[3])
: "l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).x),
"l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).y)
:);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_is_not_supported_before_SM_100__();
# endif
}
#endif // __cccl_ptx_isa >= 860
#endif // _CUDA_PTX_GENERATED_CLUSTERLAUNCHCONTROL_H_

245
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/cp_async_bulk.h
View File

@@ -1,245 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_CP_ASYNC_BULK_H_
#define _CUDA_PTX_GENERATED_CP_ASYNC_BULK_H_
/*
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes [dstMem], [srcMem], size, [smem_bar]; // PTX ISA 80, SM_90
// .dst = { .shared::cluster }
// .src = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk(
cuda::ptx::space_cluster_t,
cuda::ptx::space_global_t,
void* dstMem,
const void* srcMem,
const uint32_t& size,
uint64_t* smem_bar);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void cp_async_bulk(
::cuda::ptx::space_cluster_t,
::cuda::ptx::space_global_t,
void* __dstMem,
const void* __srcMem,
const ::cuda::std::uint32_t& __size,
::cuda::std::uint64_t* __smem_bar)
{
// __space == space_cluster (due to parameter type constraint)
// __space == space_global (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];"
:
: "r"(__as_ptr_smem(__dstMem)), "l"(__as_ptr_gmem(__srcMem)), "r"(__size), "r"(__as_ptr_smem(__smem_bar))
: "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes [dstMem], [srcMem], size, [smem_bar]; // PTX ISA 86, SM_90
// .dst = { .shared::cta }
// .src = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk(
cuda::ptx::space_shared_t,
cuda::ptx::space_global_t,
void* dstMem,
const void* srcMem,
const uint32_t& size,
uint64_t* smem_bar);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void cp_async_bulk(
::cuda::ptx::space_shared_t,
::cuda::ptx::space_global_t,
void* __dstMem,
const void* __srcMem,
const ::cuda::std::uint32_t& __size,
::cuda::std::uint64_t* __smem_bar)
{
// __space == space_shared (due to parameter type constraint)
// __space == space_global (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm("cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];"
:
: "r"(__as_ptr_smem(__dstMem)), "l"(__as_ptr_gmem(__srcMem)), "r"(__size), "r"(__as_ptr_smem(__smem_bar))
: "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes.ignore_oob [dstMem], [srcMem], size, ignoreBytesLeft,
ignoreBytesRight, [smem_bar]; // PTX ISA 92, SM_90
// .dst = { .shared::cta }
// .src = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_ignore_oob(
cuda::ptx::space_shared_t,
cuda::ptx::space_global_t,
void* dstMem,
const void* srcMem,
const uint32_t& size,
const uint32_t& ignoreBytesLeft,
const uint32_t& ignoreBytesRight,
uint64_t* smem_bar);
*/
#if __cccl_ptx_isa >= 920
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_ignore_oob_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void cp_async_bulk_ignore_oob(
::cuda::ptx::space_shared_t,
::cuda::ptx::space_global_t,
void* __dstMem,
const void* __srcMem,
const ::cuda::std::uint32_t& __size,
const ::cuda::std::uint32_t& __ignoreBytesLeft,
const ::cuda::std::uint32_t& __ignoreBytesRight,
::cuda::std::uint64_t* __smem_bar)
{
// __space == space_shared (due to parameter type constraint)
// __space == space_global (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm("cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes.ignore_oob [%0], [%1], %2, %3, %4, [%5];"
:
: "r"(__as_ptr_smem(__dstMem)),
"l"(__as_ptr_gmem(__srcMem)),
"r"(__size),
"r"(__ignoreBytesLeft),
"r"(__ignoreBytesRight),
"r"(__as_ptr_smem(__smem_bar))
: "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_cp_async_bulk_ignore_oob_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 920
/*
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes [dstMem], [srcMem], size, [rdsmem_bar]; // PTX ISA 80, SM_90
// .dst = { .shared::cluster }
// .src = { .shared::cta }
template <typename = void>
__device__ static inline void cp_async_bulk(
cuda::ptx::space_cluster_t,
cuda::ptx::space_shared_t,
void* dstMem,
const void* srcMem,
const uint32_t& size,
uint64_t* rdsmem_bar);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void cp_async_bulk(
::cuda::ptx::space_cluster_t,
::cuda::ptx::space_shared_t,
void* __dstMem,
const void* __srcMem,
const ::cuda::std::uint32_t& __size,
::cuda::std::uint64_t* __rdsmem_bar)
{
// __space == space_cluster (due to parameter type constraint)
// __space == space_shared (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm("cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];"
:
: "r"(__as_ptr_remote_dsmem(__dstMem)),
"r"(__as_ptr_smem(__srcMem)),
"r"(__size),
"r"(__as_ptr_remote_dsmem(__rdsmem_bar))
: "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// cp.async.bulk.dst.src.bulk_group [dstMem], [srcMem], size; // PTX ISA 80, SM_90
// .dst = { .global }
// .src = { .shared::cta }
template <typename = void>
__device__ static inline void cp_async_bulk(
cuda::ptx::space_global_t,
cuda::ptx::space_shared_t,
void* dstMem,
const void* srcMem,
const uint32_t& size);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void cp_async_bulk(
::cuda::ptx::space_global_t,
::cuda::ptx::space_shared_t,
void* __dstMem,
const void* __srcMem,
const ::cuda::std::uint32_t& __size)
{
// __space == space_global (due to parameter type constraint)
// __space == space_shared (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm("cp.async.bulk.global.shared::cta.bulk_group [%0], [%1], %2;"
:
: "l"(__as_ptr_gmem(__dstMem)), "r"(__as_ptr_smem(__srcMem)), "r"(__size)
: "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// cp.async.bulk.dst.src.bulk_group.cp_mask [dstMem], [srcMem], size, byteMask; // PTX ISA 86, SM_100
// .dst = { .global }
// .src = { .shared::cta }
template <typename = void>
__device__ static inline void cp_async_bulk_cp_mask(
cuda::ptx::space_global_t,
cuda::ptx::space_shared_t,
void* dstMem,
const void* srcMem,
const uint32_t& size,
const uint16_t& byteMask);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_cp_mask_is_not_supported_before_SM_100__();
template <typename = void>
_CCCL_DEVICE static inline void cp_async_bulk_cp_mask(
::cuda::ptx::space_global_t,
::cuda::ptx::space_shared_t,
void* __dstMem,
const void* __srcMem,
const ::cuda::std::uint32_t& __size,
const ::cuda::std::uint16_t& __byteMask)
{
// __space == space_global (due to parameter type constraint)
// __space == space_shared (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
asm("cp.async.bulk.global.shared::cta.bulk_group.cp_mask [%0], [%1], %2, %3;"
:
: "l"(__as_ptr_gmem(__dstMem)), "r"(__as_ptr_smem(__srcMem)), "r"(__size), "h"(__byteMask)
: "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_cp_async_bulk_cp_mask_is_not_supported_before_SM_100__();
# endif
}
#endif // __cccl_ptx_isa >= 860
#endif // _CUDA_PTX_GENERATED_CP_ASYNC_BULK_H_

25
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/cp_async_bulk_commit_group.h
View File

@@ -1,25 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_CP_ASYNC_BULK_COMMIT_GROUP_H_
#define _CUDA_PTX_GENERATED_CP_ASYNC_BULK_COMMIT_GROUP_H_
/*
// cp.async.bulk.commit_group; // PTX ISA 80, SM_90
template <typename = void>
__device__ static inline void cp_async_bulk_commit_group();
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_commit_group_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void cp_async_bulk_commit_group()
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm volatile("cp.async.bulk.commit_group;" : : :);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_cp_async_bulk_commit_group_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 800
#endif // _CUDA_PTX_GENERATED_CP_ASYNC_BULK_COMMIT_GROUP_H_

52
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/cp_async_bulk_multicast.h
View File

@@ -1,52 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_CP_ASYNC_BULK_MULTICAST_H_
#define _CUDA_PTX_GENERATED_CP_ASYNC_BULK_MULTICAST_H_
/*
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [srcMem], size, [smem_bar], ctaMask;
// PTX ISA 80, SM_90a, SM_100a, SM_110a
// .dst = { .shared::cluster }
// .src = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk(
cuda::ptx::space_cluster_t,
cuda::ptx::space_global_t,
void* dstMem,
const void* srcMem,
const uint32_t& size,
uint64_t* smem_bar,
const uint16_t& ctaMask);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_is_only_supported_on_SM_90a_100a_110a__();
template <typename = void>
_CCCL_DEVICE static inline void cp_async_bulk(
::cuda::ptx::space_cluster_t,
::cuda::ptx::space_global_t,
void* __dstMem,
const void* __srcMem,
const ::cuda::std::uint32_t& __size,
::cuda::std::uint64_t* __smem_bar,
const ::cuda::std::uint16_t& __ctaMask)
{
// __space == space_cluster (due to parameter type constraint)
// __space == space_global (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
asm("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster [%0], [%1], %2, [%3], %4;"
:
: "r"(__as_ptr_smem(__dstMem)),
"l"(__as_ptr_gmem(__srcMem)),
"r"(__size),
"r"(__as_ptr_smem(__smem_bar)),
"h"(__ctaMask)
: "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_cp_async_bulk_is_only_supported_on_SM_90a_100a_110a__();
# endif
}
#endif // __cccl_ptx_isa >= 800
#endif // _CUDA_PTX_GENERATED_CP_ASYNC_BULK_MULTICAST_H_

46
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/cp_async_bulk_wait_group.h
View File

@@ -1,46 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_CP_ASYNC_BULK_WAIT_GROUP_H_
#define _CUDA_PTX_GENERATED_CP_ASYNC_BULK_WAIT_GROUP_H_
/*
// cp.async.bulk.wait_group N; // PTX ISA 80, SM_90
template <int N32>
__device__ static inline void cp_async_bulk_wait_group(
cuda::ptx::n32_t<N32> N);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_wait_group_is_not_supported_before_SM_90__();
template <int _N32>
_CCCL_DEVICE static inline void cp_async_bulk_wait_group(::cuda::ptx::n32_t<_N32> __N)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm volatile("cp.async.bulk.wait_group %0;" : : "n"(__N.value) : "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_cp_async_bulk_wait_group_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// cp.async.bulk.wait_group.read N; // PTX ISA 80, SM_90
template <int N32>
__device__ static inline void cp_async_bulk_wait_group_read(
cuda::ptx::n32_t<N32> N);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_wait_group_read_is_not_supported_before_SM_90__();
template <int _N32>
_CCCL_DEVICE static inline void cp_async_bulk_wait_group_read(::cuda::ptx::n32_t<_N32> __N)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm volatile("cp.async.bulk.wait_group.read %0;" : : "n"(__N.value) : "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_cp_async_bulk_wait_group_read_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 800
#endif // _CUDA_PTX_GENERATED_CP_ASYNC_BULK_WAIT_GROUP_H_

36
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/elect_sync.h
View File

@@ -1,36 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_ELECT_SYNC_H_
#define _CUDA_PTX_GENERATED_ELECT_SYNC_H_
/*
// elect.sync _|is_elected, membermask; // PTX ISA 80, SM_90
template <typename = void>
__device__ static inline bool elect_sync(
const uint32_t& membermask);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_elect_sync_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline bool elect_sync(const ::cuda::std::uint32_t& __membermask)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __is_elected;
asm volatile(
"{\n\t .reg .pred P_OUT; \n\t"
"elect.sync _|P_OUT, %1;\n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__is_elected)
: "r"(__membermask)
:);
return static_cast<bool>(__is_elected);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_elect_sync_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 800
#endif // _CUDA_PTX_GENERATED_ELECT_SYNC_H_

212
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/fence.h
View File

@@ -1,212 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_FENCE_H_
#define _CUDA_PTX_GENERATED_FENCE_H_
/*
// fence.sem.scope; // 1. PTX ISA 60, SM_70
// .sem = { .sc }
// .scope = { .cta, .gpu, .sys }
template <cuda::ptx::dot_scope Scope>
__device__ static inline void fence(
cuda::ptx::sem_sc_t,
cuda::ptx::scope_t<Scope> scope);
*/
#if __cccl_ptx_isa >= 600
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_is_not_supported_before_SM_70__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline void fence(::cuda::ptx::sem_sc_t, ::cuda::ptx::scope_t<_Scope> __scope)
{
// __sem == sem_sc (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_gpu || __scope == scope_sys);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 700
// NOLINTBEGIN(bugprone-branch-clone)
if constexpr (__scope == scope_cta)
{
asm volatile("fence.sc.cta; // 1." : : : "memory");
}
else if constexpr (__scope == scope_gpu)
{
asm volatile("fence.sc.gpu; // 1." : : : "memory");
}
else if constexpr (__scope == scope_sys)
{
asm volatile("fence.sc.sys; // 1." : : : "memory");
}
// NOLINTEND(bugprone-branch-clone)
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_is_not_supported_before_SM_70__();
# endif
}
#endif // __cccl_ptx_isa >= 600
/*
// fence.sem.scope; // 2. PTX ISA 78, SM_90
// .sem = { .sc }
// .scope = { .cluster }
template <typename = void>
__device__ static inline void fence(
cuda::ptx::sem_sc_t,
cuda::ptx::scope_cluster_t);
*/
#if __cccl_ptx_isa >= 780
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void fence(::cuda::ptx::sem_sc_t, ::cuda::ptx::scope_cluster_t)
{
// __sem == sem_sc (due to parameter type constraint)
// __scope == scope_cluster (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm volatile("fence.sc.cluster; // 2." : : : "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 780
/*
// fence.sem.scope; // 1. PTX ISA 60, SM_70
// .sem = { .acq_rel }
// .scope = { .cta, .gpu, .sys }
template <cuda::ptx::dot_scope Scope>
__device__ static inline void fence(
cuda::ptx::sem_acq_rel_t,
cuda::ptx::scope_t<Scope> scope);
*/
#if __cccl_ptx_isa >= 600
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_is_not_supported_before_SM_70__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline void fence(::cuda::ptx::sem_acq_rel_t, ::cuda::ptx::scope_t<_Scope> __scope)
{
// __sem == sem_acq_rel (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_gpu || __scope == scope_sys);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 700
// NOLINTBEGIN(bugprone-branch-clone)
if constexpr (__scope == scope_cta)
{
asm volatile("fence.acq_rel.cta; // 1." : : : "memory");
}
else if constexpr (__scope == scope_gpu)
{
asm volatile("fence.acq_rel.gpu; // 1." : : : "memory");
}
else if constexpr (__scope == scope_sys)
{
asm volatile("fence.acq_rel.sys; // 1." : : : "memory");
}
// NOLINTEND(bugprone-branch-clone)
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_is_not_supported_before_SM_70__();
# endif
}
#endif // __cccl_ptx_isa >= 600
/*
// fence.sem.scope; // 2. PTX ISA 78, SM_90
// .sem = { .acq_rel }
// .scope = { .cluster }
template <typename = void>
__device__ static inline void fence(
cuda::ptx::sem_acq_rel_t,
cuda::ptx::scope_cluster_t);
*/
#if __cccl_ptx_isa >= 780
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void fence(::cuda::ptx::sem_acq_rel_t, ::cuda::ptx::scope_cluster_t)
{
// __sem == sem_acq_rel (due to parameter type constraint)
// __scope == scope_cluster (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm volatile("fence.acq_rel.cluster; // 2." : : : "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 780
/*
// fence.sem.scope; // PTX ISA 86, SM_90
// .sem = { .acquire }
// .scope = { .cta, .cluster, .gpu, .sys }
template <cuda::ptx::dot_scope Scope>
__device__ static inline void fence(
cuda::ptx::sem_acquire_t,
cuda::ptx::scope_t<Scope> scope);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline void fence(::cuda::ptx::sem_acquire_t, ::cuda::ptx::scope_t<_Scope> __scope)
{
// __sem == sem_acquire (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster || __scope == scope_gpu || __scope == scope_sys);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
if constexpr (__scope == scope_cta)
{
asm volatile("fence.acquire.cta;" : : : "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm volatile("fence.acquire.cluster;" : : : "memory");
}
else if constexpr (__scope == scope_gpu)
{
asm volatile("fence.acquire.gpu;" : : : "memory");
}
else if constexpr (__scope == scope_sys)
{
asm volatile("fence.acquire.sys;" : : : "memory");
}
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// fence.sem.scope; // PTX ISA 86, SM_90
// .sem = { .release }
// .scope = { .cta, .cluster, .gpu, .sys }
template <cuda::ptx::dot_scope Scope>
__device__ static inline void fence(
cuda::ptx::sem_release_t,
cuda::ptx::scope_t<Scope> scope);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline void fence(::cuda::ptx::sem_release_t, ::cuda::ptx::scope_t<_Scope> __scope)
{
// __sem == sem_release (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster || __scope == scope_gpu || __scope == scope_sys);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
if constexpr (__scope == scope_cta)
{
asm volatile("fence.release.cta;" : : : "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm volatile("fence.release.cluster;" : : : "memory");
}
else if constexpr (__scope == scope_gpu)
{
asm volatile("fence.release.gpu;" : : : "memory");
}
else if constexpr (__scope == scope_sys)
{
asm volatile("fence.release.sys;" : : : "memory");
}
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 860
#endif // _CUDA_PTX_GENERATED_FENCE_H_

31
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/fence_mbarrier_init.h
View File

@@ -1,31 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_FENCE_MBARRIER_INIT_H_
#define _CUDA_PTX_GENERATED_FENCE_MBARRIER_INIT_H_
/*
// fence.mbarrier_init.sem.scope; // 3. PTX ISA 80, SM_90
// .sem = { .release }
// .scope = { .cluster }
template <typename = void>
__device__ static inline void fence_mbarrier_init(
cuda::ptx::sem_release_t,
cuda::ptx::scope_cluster_t);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_mbarrier_init_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void fence_mbarrier_init(::cuda::ptx::sem_release_t, ::cuda::ptx::scope_cluster_t)
{
// __sem == sem_release (due to parameter type constraint)
// __scope == scope_cluster (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm volatile("fence.mbarrier_init.release.cluster; // 3." : : : "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_mbarrier_init_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 800
#endif // _CUDA_PTX_GENERATED_FENCE_MBARRIER_INIT_H_

25
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/fence_proxy_alias.h
View File

@@ -1,25 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_FENCE_PROXY_ALIAS_H_
#define _CUDA_PTX_GENERATED_FENCE_PROXY_ALIAS_H_
/*
// fence.proxy.alias; // 4. PTX ISA 75, SM_70
template <typename = void>
__device__ static inline void fence_proxy_alias();
*/
#if __cccl_ptx_isa >= 750
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_proxy_alias_is_not_supported_before_SM_70__();
template <typename = void>
_CCCL_DEVICE static inline void fence_proxy_alias()
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 700
asm volatile("fence.proxy.alias; // 4." : : : "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_proxy_alias_is_not_supported_before_SM_70__();
# endif
}
#endif // __cccl_ptx_isa >= 750
#endif // _CUDA_PTX_GENERATED_FENCE_PROXY_ALIAS_H_

58
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/fence_proxy_async.h
View File

@@ -1,58 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_FENCE_PROXY_ASYNC_H_
#define _CUDA_PTX_GENERATED_FENCE_PROXY_ASYNC_H_
/*
// fence.proxy.async; // 5. PTX ISA 80, SM_90
template <typename = void>
__device__ static inline void fence_proxy_async();
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_proxy_async_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void fence_proxy_async()
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm volatile("fence.proxy.async; // 5." : : : "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_proxy_async_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// fence.proxy.async.space; // 6. PTX ISA 80, SM_90
// .space = { .global, .shared::cluster, .shared::cta }
template <cuda::ptx::dot_space Space>
__device__ static inline void fence_proxy_async(
cuda::ptx::space_t<Space> space);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_proxy_async_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_space _Space>
_CCCL_DEVICE static inline void fence_proxy_async(::cuda::ptx::space_t<_Space> __space)
{
static_assert(__space == space_global || __space == space_cluster || __space == space_shared);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
if constexpr (__space == space_global)
{
asm volatile("fence.proxy.async.global; // 6." : : : "memory");
}
else if constexpr (__space == space_cluster)
{
asm volatile("fence.proxy.async.shared::cluster; // 6." : : : "memory");
}
else if constexpr (__space == space_shared)
{
asm volatile("fence.proxy.async.shared::cta; // 6." : : : "memory");
}
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_proxy_async_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 800
#endif // _CUDA_PTX_GENERATED_FENCE_PROXY_ASYNC_H_

64
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/fence_proxy_async_generic_sync_restrict.h
View File

@@ -1,64 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_FENCE_PROXY_ASYNC_GENERIC_SYNC_RESTRICT_H_
#define _CUDA_PTX_GENERATED_FENCE_PROXY_ASYNC_GENERIC_SYNC_RESTRICT_H_
/*
// fence.proxy.async::generic.sem.sync_restrict::space.scope; // PTX ISA 86, SM_90
// .sem = { .acquire }
// .space = { .shared::cluster }
// .scope = { .cluster }
template <typename = void>
__device__ static inline void fence_proxy_async_generic_sync_restrict(
cuda::ptx::sem_acquire_t,
cuda::ptx::space_cluster_t,
cuda::ptx::scope_cluster_t);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_proxy_async_generic_sync_restrict_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void fence_proxy_async_generic_sync_restrict(
::cuda::ptx::sem_acquire_t, ::cuda::ptx::space_cluster_t, ::cuda::ptx::scope_cluster_t)
{
// __sem == sem_acquire (due to parameter type constraint)
// __space == space_cluster (due to parameter type constraint)
// __scope == scope_cluster (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm volatile("fence.proxy.async::generic.acquire.sync_restrict::shared::cluster.cluster;" : : : "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_proxy_async_generic_sync_restrict_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// fence.proxy.async::generic.sem.sync_restrict::space.scope; // PTX ISA 86, SM_90
// .sem = { .release }
// .space = { .shared::cta }
// .scope = { .cluster }
template <typename = void>
__device__ static inline void fence_proxy_async_generic_sync_restrict(
cuda::ptx::sem_release_t,
cuda::ptx::space_shared_t,
cuda::ptx::scope_cluster_t);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_proxy_async_generic_sync_restrict_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void fence_proxy_async_generic_sync_restrict(
::cuda::ptx::sem_release_t, ::cuda::ptx::space_shared_t, ::cuda::ptx::scope_cluster_t)
{
// __sem == sem_release (due to parameter type constraint)
// __space == space_shared (due to parameter type constraint)
// __scope == scope_cluster (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm volatile("fence.proxy.async::generic.release.sync_restrict::shared::cta.cluster;" : : : "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_proxy_async_generic_sync_restrict_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 860
#endif // _CUDA_PTX_GENERATED_FENCE_PROXY_ASYNC_GENERIC_SYNC_RESTRICT_H_

102
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/fence_proxy_tensormap_generic.h
View File

@@ -1,102 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_FENCE_PROXY_TENSORMAP_GENERIC_H_
#define _CUDA_PTX_GENERATED_FENCE_PROXY_TENSORMAP_GENERIC_H_
/*
// fence.proxy.tensormap::generic.release.scope; // 7. PTX ISA 83, SM_90
// .sem = { .release }
// .scope = { .cta, .cluster, .gpu, .sys }
template <cuda::ptx::dot_scope Scope>
__device__ static inline void fence_proxy_tensormap_generic(
cuda::ptx::sem_release_t,
cuda::ptx::scope_t<Scope> scope);
*/
#if __cccl_ptx_isa >= 830
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_proxy_tensormap_generic_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline void
fence_proxy_tensormap_generic(::cuda::ptx::sem_release_t, ::cuda::ptx::scope_t<_Scope> __scope)
{
// __sem == sem_release (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster || __scope == scope_gpu || __scope == scope_sys);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
if constexpr (__scope == scope_cta)
{
asm volatile("fence.proxy.tensormap::generic.release.cta; // 7." : : : "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm volatile("fence.proxy.tensormap::generic.release.cluster; // 7." : : : "memory");
}
else if constexpr (__scope == scope_gpu)
{
asm volatile("fence.proxy.tensormap::generic.release.gpu; // 7." : : : "memory");
}
else if constexpr (__scope == scope_sys)
{
asm volatile("fence.proxy.tensormap::generic.release.sys; // 7." : : : "memory");
}
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_proxy_tensormap_generic_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 830
/*
// fence.proxy.tensormap::generic.sem.scope [addr], size; // 8. PTX ISA 83, SM_90
// .sem = { .acquire }
// .scope = { .cta, .cluster, .gpu, .sys }
template <int N32, cuda::ptx::dot_scope Scope>
__device__ static inline void fence_proxy_tensormap_generic(
cuda::ptx::sem_acquire_t,
cuda::ptx::scope_t<Scope> scope,
const void* addr,
cuda::ptx::n32_t<N32> size);
*/
#if __cccl_ptx_isa >= 830
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_proxy_tensormap_generic_is_not_supported_before_SM_90__();
template <int _N32, ::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline void fence_proxy_tensormap_generic(
::cuda::ptx::sem_acquire_t, ::cuda::ptx::scope_t<_Scope> __scope, const void* __addr, ::cuda::ptx::n32_t<_N32> __size)
{
// __sem == sem_acquire (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster || __scope == scope_gpu || __scope == scope_sys);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
if constexpr (__scope == scope_cta)
{
asm volatile("fence.proxy.tensormap::generic.acquire.cta [%0], %1; // 8."
:
: "l"(__addr), "n"(__size.value)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm volatile("fence.proxy.tensormap::generic.acquire.cluster [%0], %1; // 8."
:
: "l"(__addr), "n"(__size.value)
: "memory");
}
else if constexpr (__scope == scope_gpu)
{
asm volatile("fence.proxy.tensormap::generic.acquire.gpu [%0], %1; // 8."
:
: "l"(__addr), "n"(__size.value)
: "memory");
}
else if constexpr (__scope == scope_sys)
{
asm volatile("fence.proxy.tensormap::generic.acquire.sys [%0], %1; // 8."
:
: "l"(__addr), "n"(__size.value)
: "memory");
}
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_proxy_tensormap_generic_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 830
#endif // _CUDA_PTX_GENERATED_FENCE_PROXY_TENSORMAP_GENERIC_H_

64
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/fence_sync_restrict.h
View File

@@ -1,64 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_FENCE_SYNC_RESTRICT_H_
#define _CUDA_PTX_GENERATED_FENCE_SYNC_RESTRICT_H_
/*
// fence.sem.sync_restrict::space.scope; // PTX ISA 86, SM_90
// .sem = { .acquire }
// .space = { .shared::cluster }
// .scope = { .cluster }
template <typename = void>
__device__ static inline void fence_sync_restrict(
cuda::ptx::sem_acquire_t,
cuda::ptx::space_cluster_t,
cuda::ptx::scope_cluster_t);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_sync_restrict_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void
fence_sync_restrict(::cuda::ptx::sem_acquire_t, ::cuda::ptx::space_cluster_t, ::cuda::ptx::scope_cluster_t)
{
// __sem == sem_acquire (due to parameter type constraint)
// __space == space_cluster (due to parameter type constraint)
// __scope == scope_cluster (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm volatile("fence.acquire.sync_restrict::shared::cluster.cluster;" : : : "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_sync_restrict_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// fence.sem.sync_restrict::space.scope; // PTX ISA 86, SM_90
// .sem = { .release }
// .space = { .shared::cta }
// .scope = { .cluster }
template <typename = void>
__device__ static inline void fence_sync_restrict(
cuda::ptx::sem_release_t,
cuda::ptx::space_shared_t,
cuda::ptx::scope_cluster_t);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_sync_restrict_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void
fence_sync_restrict(::cuda::ptx::sem_release_t, ::cuda::ptx::space_shared_t, ::cuda::ptx::scope_cluster_t)
{
// __sem == sem_release (due to parameter type constraint)
// __space == space_shared (due to parameter type constraint)
// __scope == scope_cluster (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm volatile("fence.release.sync_restrict::shared::cta.cluster;" : : : "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_fence_sync_restrict_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 860
#endif // _CUDA_PTX_GENERATED_FENCE_SYNC_RESTRICT_H_

399
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/mbarrier_arrive.h
View File

@@ -1,399 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_H_
#define _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_H_
/*
// mbarrier.arrive.shared.b64 state, [addr]; // 1. PTX ISA 70, SM_80
template <typename = void>
__device__ static inline uint64_t mbarrier_arrive(
uint64_t* addr);
*/
#if __cccl_ptx_isa >= 700
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_80__();
template <typename = void>
_CCCL_DEVICE static inline ::cuda::std::uint64_t mbarrier_arrive(::cuda::std::uint64_t* __addr)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 800
::cuda::std::uint64_t __state;
asm("mbarrier.arrive.shared.b64 %0, [%1]; // 1. "
: "=l"(__state)
: "r"(__as_ptr_smem(__addr))
: "memory");
return __state;
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_80__();
return 0;
# endif
}
#endif // __cccl_ptx_isa >= 700
/*
// mbarrier.arrive.shared::cta.b64 state, [addr], count; // 2. PTX ISA 78, SM_90
template <typename = void>
__device__ static inline uint64_t mbarrier_arrive(
uint64_t* addr,
const uint32_t& count);
*/
#if __cccl_ptx_isa >= 780
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline ::cuda::std::uint64_t
mbarrier_arrive(::cuda::std::uint64_t* __addr, const ::cuda::std::uint32_t& __count)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint64_t __state;
asm("mbarrier.arrive.shared::cta.b64 %0, [%1], %2; // 2. "
: "=l"(__state)
: "r"(__as_ptr_smem(__addr)), "r"(__count)
: "memory");
return __state;
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
return 0;
# endif
}
#endif // __cccl_ptx_isa >= 780
/*
// mbarrier.arrive.sem.scope.space.b64 state, [addr]; // 3a. PTX ISA 80, SM_90
// .sem = { .release }
// .scope = { .cta, .cluster }
// .space = { .shared::cta }
template <cuda::ptx::dot_scope Scope>
__device__ static inline uint64_t mbarrier_arrive(
cuda::ptx::sem_release_t,
cuda::ptx::scope_t<Scope> scope,
cuda::ptx::space_shared_t,
uint64_t* addr);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline ::cuda::std::uint64_t mbarrier_arrive(
::cuda::ptx::sem_release_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::ptx::space_shared_t,
::cuda::std::uint64_t* __addr)
{
// __sem == sem_release (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
// __space == space_shared (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint64_t __state;
if constexpr (__scope == scope_cta)
{
asm("mbarrier.arrive.release.cta.shared::cta.b64 %0, [%1]; // 3a. "
: "=l"(__state)
: "r"(__as_ptr_smem(__addr))
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("mbarrier.arrive.release.cluster.shared::cta.b64 %0, [%1]; // 3a. "
: "=l"(__state)
: "r"(__as_ptr_smem(__addr))
: "memory");
}
return __state;
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
return 0;
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// mbarrier.arrive.sem.scope.space.b64 state, [addr], count; // 3b. PTX ISA 80, SM_90
// .sem = { .release }
// .scope = { .cta, .cluster }
// .space = { .shared::cta }
template <cuda::ptx::dot_scope Scope>
__device__ static inline uint64_t mbarrier_arrive(
cuda::ptx::sem_release_t,
cuda::ptx::scope_t<Scope> scope,
cuda::ptx::space_shared_t,
uint64_t* addr,
const uint32_t& count);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline ::cuda::std::uint64_t mbarrier_arrive(
::cuda::ptx::sem_release_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::ptx::space_shared_t,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __count)
{
// __sem == sem_release (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
// __space == space_shared (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint64_t __state;
if constexpr (__scope == scope_cta)
{
asm("mbarrier.arrive.release.cta.shared::cta.b64 %0, [%1], %2; // 3b. "
: "=l"(__state)
: "r"(__as_ptr_smem(__addr)), "r"(__count)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("mbarrier.arrive.release.cluster.shared::cta.b64 %0, [%1], %2; // 3b. "
: "=l"(__state)
: "r"(__as_ptr_smem(__addr)), "r"(__count)
: "memory");
}
return __state;
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
return 0;
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// mbarrier.arrive.sem.scope.space.b64 _, [addr]; // 4a. PTX ISA 80, SM_90
// .sem = { .release }
// .scope = { .cluster }
// .space = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive(
cuda::ptx::sem_release_t,
cuda::ptx::scope_cluster_t,
cuda::ptx::space_cluster_t,
uint64_t* addr);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void mbarrier_arrive(
::cuda::ptx::sem_release_t, ::cuda::ptx::scope_cluster_t, ::cuda::ptx::space_cluster_t, ::cuda::std::uint64_t* __addr)
{
// __sem == sem_release (due to parameter type constraint)
// __scope == scope_cluster (due to parameter type constraint)
// __space == space_cluster (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm("mbarrier.arrive.release.cluster.shared::cluster.b64 _, [%0]; // 4a. "
:
: "r"(__as_ptr_remote_dsmem(__addr))
: "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// mbarrier.arrive.sem.scope.space.b64 _, [addr], count; // 4b. PTX ISA 80, SM_90
// .sem = { .release }
// .scope = { .cluster }
// .space = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive(
cuda::ptx::sem_release_t,
cuda::ptx::scope_cluster_t,
cuda::ptx::space_cluster_t,
uint64_t* addr,
const uint32_t& count);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void mbarrier_arrive(
::cuda::ptx::sem_release_t,
::cuda::ptx::scope_cluster_t,
::cuda::ptx::space_cluster_t,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __count)
{
// __sem == sem_release (due to parameter type constraint)
// __scope == scope_cluster (due to parameter type constraint)
// __space == space_cluster (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm("mbarrier.arrive.release.cluster.shared::cluster.b64 _, [%0], %1; // 4b. "
:
: "r"(__as_ptr_remote_dsmem(__addr)), "r"(__count)
: "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// mbarrier.arrive.sem.scope.space.b64 state, [addr], count; // PTX ISA 86, SM_90
// .sem = { .relaxed }
// .scope = { .cta, .cluster }
// .space = { .shared::cta }
template <cuda::ptx::dot_scope Scope>
__device__ static inline uint64_t mbarrier_arrive(
cuda::ptx::sem_relaxed_t,
cuda::ptx::scope_t<Scope> scope,
cuda::ptx::space_shared_t,
uint64_t* addr,
const uint32_t& count);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline ::cuda::std::uint64_t mbarrier_arrive(
::cuda::ptx::sem_relaxed_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::ptx::space_shared_t,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __count)
{
// __sem == sem_relaxed (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
// __space == space_shared (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint64_t __state;
if constexpr (__scope == scope_cta)
{
asm("mbarrier.arrive.relaxed.cta.shared::cta.b64 %0, [%1], %2;"
: "=l"(__state)
: "r"(__as_ptr_smem(__addr)), "r"(__count)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("mbarrier.arrive.relaxed.cluster.shared::cta.b64 %0, [%1], %2;"
: "=l"(__state)
: "r"(__as_ptr_smem(__addr)), "r"(__count)
: "memory");
}
return __state;
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
return 0;
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// mbarrier.arrive.sem.scope.space.b64 state, [addr]; // PTX ISA 86, SM_90
// .sem = { .relaxed }
// .scope = { .cta, .cluster }
// .space = { .shared::cta }
template <cuda::ptx::dot_scope Scope>
__device__ static inline uint64_t mbarrier_arrive(
cuda::ptx::sem_relaxed_t,
cuda::ptx::scope_t<Scope> scope,
cuda::ptx::space_shared_t,
uint64_t* addr);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline ::cuda::std::uint64_t mbarrier_arrive(
::cuda::ptx::sem_relaxed_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::ptx::space_shared_t,
::cuda::std::uint64_t* __addr)
{
// __sem == sem_relaxed (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
// __space == space_shared (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint64_t __state;
if constexpr (__scope == scope_cta)
{
asm("mbarrier.arrive.relaxed.cta.shared::cta.b64 %0, [%1];"
: "=l"(__state)
: "r"(__as_ptr_smem(__addr))
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("mbarrier.arrive.relaxed.cluster.shared::cta.b64 %0, [%1];"
: "=l"(__state)
: "r"(__as_ptr_smem(__addr))
: "memory");
}
return __state;
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
return 0;
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// mbarrier.arrive.sem.scope.space.b64 _, [addr], count; // PTX ISA 86, SM_90
// .sem = { .relaxed }
// .scope = { .cluster }
// .space = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive(
cuda::ptx::sem_relaxed_t,
cuda::ptx::scope_cluster_t,
cuda::ptx::space_cluster_t,
uint64_t* addr,
const uint32_t& count);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void mbarrier_arrive(
::cuda::ptx::sem_relaxed_t,
::cuda::ptx::scope_cluster_t,
::cuda::ptx::space_cluster_t,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __count)
{
// __sem == sem_relaxed (due to parameter type constraint)
// __scope == scope_cluster (due to parameter type constraint)
// __space == space_cluster (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm("mbarrier.arrive.relaxed.cluster.shared::cluster.b64 _, [%0], %1;"
:
: "r"(__as_ptr_smem(__addr)), "r"(__count)
: "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// mbarrier.arrive.sem.scope.space.b64 _, [addr]; // PTX ISA 86, SM_90
// .sem = { .relaxed }
// .scope = { .cluster }
// .space = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive(
cuda::ptx::sem_relaxed_t,
cuda::ptx::scope_cluster_t,
cuda::ptx::space_cluster_t,
uint64_t* addr);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void mbarrier_arrive(
::cuda::ptx::sem_relaxed_t, ::cuda::ptx::scope_cluster_t, ::cuda::ptx::space_cluster_t, ::cuda::std::uint64_t* __addr)
{
// __sem == sem_relaxed (due to parameter type constraint)
// __scope == scope_cluster (due to parameter type constraint)
// __space == space_cluster (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm("mbarrier.arrive.relaxed.cluster.shared::cluster.b64 _, [%0];" : : "r"(__as_ptr_smem(__addr)) : "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 860
#endif // _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_H_

184
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/mbarrier_arrive_expect_tx.h
View File

@@ -1,184 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_EXPECT_TX_H_
#define _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_EXPECT_TX_H_
/*
// mbarrier.arrive.expect_tx.sem.scope.space.b64 state, [addr], tx_count; // 8. PTX ISA 80, SM_90
// .sem = { .release }
// .scope = { .cta, .cluster }
// .space = { .shared::cta }
template <cuda::ptx::dot_scope Scope>
__device__ static inline uint64_t mbarrier_arrive_expect_tx(
cuda::ptx::sem_release_t,
cuda::ptx::scope_t<Scope> scope,
cuda::ptx::space_shared_t,
uint64_t* addr,
const uint32_t& tx_count);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline ::cuda::std::uint64_t mbarrier_arrive_expect_tx(
::cuda::ptx::sem_release_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::ptx::space_shared_t,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __tx_count)
{
// __sem == sem_release (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
// __space == space_shared (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint64_t __state;
if constexpr (__scope == scope_cta)
{
asm("mbarrier.arrive.expect_tx.release.cta.shared::cta.b64 %0, [%1], %2; // 8. "
: "=l"(__state)
: "r"(__as_ptr_smem(__addr)), "r"(__tx_count)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("mbarrier.arrive.expect_tx.release.cluster.shared::cta.b64 %0, [%1], %2; // 8. "
: "=l"(__state)
: "r"(__as_ptr_smem(__addr)), "r"(__tx_count)
: "memory");
}
return __state;
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
return 0;
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// mbarrier.arrive.expect_tx.sem.scope.space.b64 _, [addr], tx_count; // 9. PTX ISA 80, SM_90
// .sem = { .release }
// .scope = { .cluster }
// .space = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive_expect_tx(
cuda::ptx::sem_release_t,
cuda::ptx::scope_cluster_t,
cuda::ptx::space_cluster_t,
uint64_t* addr,
const uint32_t& tx_count);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void mbarrier_arrive_expect_tx(
::cuda::ptx::sem_release_t,
::cuda::ptx::scope_cluster_t,
::cuda::ptx::space_cluster_t,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __tx_count)
{
// __sem == sem_release (due to parameter type constraint)
// __scope == scope_cluster (due to parameter type constraint)
// __space == space_cluster (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm("mbarrier.arrive.expect_tx.release.cluster.shared::cluster.b64 _, [%0], %1; // 9. "
:
: "r"(__as_ptr_remote_dsmem(__addr)), "r"(__tx_count)
: "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// mbarrier.arrive.expect_tx.sem.scope.space.b64 state, [addr], txCount; // PTX ISA 86, SM_90
// .sem = { .relaxed }
// .scope = { .cta, .cluster }
// .space = { .shared::cta }
template <cuda::ptx::dot_scope Scope>
__device__ static inline uint64_t mbarrier_arrive_expect_tx(
cuda::ptx::sem_relaxed_t,
cuda::ptx::scope_t<Scope> scope,
cuda::ptx::space_shared_t,
uint64_t* addr,
const uint32_t& txCount);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline ::cuda::std::uint64_t mbarrier_arrive_expect_tx(
::cuda::ptx::sem_relaxed_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::ptx::space_shared_t,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __txCount)
{
// __sem == sem_relaxed (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
// __space == space_shared (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint64_t __state;
if constexpr (__scope == scope_cta)
{
asm("mbarrier.arrive.expect_tx.relaxed.cta.shared::cta.b64 %0, [%1], %2;"
: "=l"(__state)
: "r"(__as_ptr_dsmem(__addr)), "r"(__txCount)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("mbarrier.arrive.expect_tx.relaxed.cluster.shared::cta.b64 %0, [%1], %2;"
: "=l"(__state)
: "r"(__as_ptr_dsmem(__addr)), "r"(__txCount)
: "memory");
}
return __state;
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
return 0;
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// mbarrier.arrive.expect_tx.sem.scope.space.b64 _, [addr], txCount; // PTX ISA 86, SM_90
// .sem = { .relaxed }
// .scope = { .cluster }
// .space = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive_expect_tx(
cuda::ptx::sem_relaxed_t,
cuda::ptx::scope_cluster_t,
cuda::ptx::space_cluster_t,
uint64_t* addr,
const uint32_t& txCount);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline void mbarrier_arrive_expect_tx(
::cuda::ptx::sem_relaxed_t,
::cuda::ptx::scope_cluster_t,
::cuda::ptx::space_cluster_t,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __txCount)
{
// __sem == sem_relaxed (due to parameter type constraint)
// __scope == scope_cluster (due to parameter type constraint)
// __space == space_cluster (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
asm("mbarrier.arrive.expect_tx.relaxed.cluster.shared::cluster.b64 _, [%0], %1;"
:
: "r"(__as_ptr_dsmem(__addr)), "r"(__txCount)
: "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 860
#endif // _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_EXPECT_TX_H_

34
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/mbarrier_arrive_no_complete.h
View File

@@ -1,34 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_NO_COMPLETE_H_
#define _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_NO_COMPLETE_H_
/*
// mbarrier.arrive.noComplete.shared.b64 state, [addr], count; // 5. PTX ISA 70, SM_80
template <typename = void>
__device__ static inline uint64_t mbarrier_arrive_no_complete(
uint64_t* addr,
const uint32_t& count);
*/
#if __cccl_ptx_isa >= 700
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_no_complete_is_not_supported_before_SM_80__();
template <typename = void>
_CCCL_DEVICE static inline ::cuda::std::uint64_t
mbarrier_arrive_no_complete(::cuda::std::uint64_t* __addr, const ::cuda::std::uint32_t& __count)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 800
::cuda::std::uint64_t __state;
asm("mbarrier.arrive.noComplete.shared.b64 %0, [%1], %2; // 5. "
: "=l"(__state)
: "r"(__as_ptr_smem(__addr)), "r"(__count)
: "memory");
return __state;
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_arrive_no_complete_is_not_supported_before_SM_80__();
return 0;
# endif
}
#endif // __cccl_ptx_isa >= 700
#endif // _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_NO_COMPLETE_H_

102
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/mbarrier_expect_tx.h
View File

@@ -1,102 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_MBARRIER_EXPECT_TX_H_
#define _CUDA_PTX_GENERATED_MBARRIER_EXPECT_TX_H_
/*
// mbarrier.expect_tx.sem.scope.space.b64 [addr], txCount; // 1. PTX ISA 80, SM_90
// .sem = { .relaxed }
// .scope = { .cta, .cluster }
// .space = { .shared::cta }
template <cuda::ptx::dot_scope Scope>
__device__ static inline void mbarrier_expect_tx(
cuda::ptx::sem_relaxed_t,
cuda::ptx::scope_t<Scope> scope,
cuda::ptx::space_shared_t,
uint64_t* addr,
uint32_t txCount);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_expect_tx_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline void mbarrier_expect_tx(
::cuda::ptx::sem_relaxed_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::ptx::space_shared_t,
::cuda::std::uint64_t* __addr,
::cuda::std::uint32_t __txCount)
{
// __sem == sem_relaxed (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
// __space == space_shared (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
if constexpr (__scope == scope_cta)
{
asm("mbarrier.expect_tx.relaxed.cta.shared::cta.b64 [%0], %1; // 1."
:
: "r"(__as_ptr_smem(__addr)), "r"(__txCount)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("mbarrier.expect_tx.relaxed.cluster.shared::cta.b64 [%0], %1; // 1."
:
: "r"(__as_ptr_smem(__addr)), "r"(__txCount)
: "memory");
}
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_expect_tx_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// mbarrier.expect_tx.sem.scope.space.b64 [addr], txCount; // 2. PTX ISA 80, SM_90
// .sem = { .relaxed }
// .scope = { .cta, .cluster }
// .space = { .shared::cluster }
template <cuda::ptx::dot_scope Scope>
__device__ static inline void mbarrier_expect_tx(
cuda::ptx::sem_relaxed_t,
cuda::ptx::scope_t<Scope> scope,
cuda::ptx::space_cluster_t,
uint64_t* addr,
uint32_t txCount);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_expect_tx_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline void mbarrier_expect_tx(
::cuda::ptx::sem_relaxed_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::ptx::space_cluster_t,
::cuda::std::uint64_t* __addr,
::cuda::std::uint32_t __txCount)
{
// __sem == sem_relaxed (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
// __space == space_cluster (due to parameter type constraint)
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
if constexpr (__scope == scope_cta)
{
asm("mbarrier.expect_tx.relaxed.cta.shared::cluster.b64 [%0], %1; // 2."
:
: "r"(__as_ptr_dsmem(__addr)), "r"(__txCount)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("mbarrier.expect_tx.relaxed.cluster.shared::cluster.b64 [%0], %1; // 2."
:
: "r"(__as_ptr_dsmem(__addr)), "r"(__txCount)
: "memory");
}
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_expect_tx_is_not_supported_before_SM_90__();
# endif
}
#endif // __cccl_ptx_isa >= 800
#endif // _CUDA_PTX_GENERATED_MBARRIER_EXPECT_TX_H_

27
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/mbarrier_init.h
View File

@@ -1,27 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_MBARRIER_INIT_H_
#define _CUDA_PTX_GENERATED_MBARRIER_INIT_H_
/*
// mbarrier.init.shared.b64 [addr], count; // PTX ISA 70, SM_80
template <typename = void>
__device__ static inline void mbarrier_init(
uint64_t* addr,
const uint32_t& count);
*/
#if __cccl_ptx_isa >= 700
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_init_is_not_supported_before_SM_80__();
template <typename = void>
_CCCL_DEVICE static inline void mbarrier_init(::cuda::std::uint64_t* __addr, const ::cuda::std::uint32_t& __count)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 800
asm("mbarrier.init.shared.b64 [%0], %1;" : : "r"(__as_ptr_smem(__addr)), "r"(__count) : "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_init_is_not_supported_before_SM_80__();
# endif
}
#endif // __cccl_ptx_isa >= 700
#endif // _CUDA_PTX_GENERATED_MBARRIER_INIT_H_

26
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/mbarrier_inval.h
View File

@@ -1,26 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_MBARRIER_INVAL_H_
#define _CUDA_PTX_GENERATED_MBARRIER_INVAL_H_
/*
// mbarrier.inval.shared.b64 [addr]; // PTX ISA 70, SM_80
template <typename = void>
__device__ static inline void mbarrier_inval(
uint64_t* addr);
*/
#if __cccl_ptx_isa >= 700
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_inval_is_not_supported_before_SM_80__();
template <typename = void>
_CCCL_DEVICE static inline void mbarrier_inval(::cuda::std::uint64_t* __addr)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 800
asm("mbarrier.inval.shared.b64 [%0];" : : "r"(__as_ptr_smem(__addr)) : "memory");
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_inval_is_not_supported_before_SM_80__();
# endif
}
#endif // __cccl_ptx_isa >= 700
#endif // _CUDA_PTX_GENERATED_MBARRIER_INVAL_H_

143
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/mbarrier_test_wait.h
View File

@@ -1,143 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_MBARRIER_TEST_WAIT_H_
#define _CUDA_PTX_GENERATED_MBARRIER_TEST_WAIT_H_
/*
// mbarrier.test_wait.shared.b64 waitComplete, [addr], state; // 1. PTX
ISA 70, SM_80 template <typename = void>
__device__ static inline bool mbarrier_test_wait(
uint64_t* addr,
const uint64_t& state);
*/
#if __cccl_ptx_isa >= 700
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_80__();
template <typename = void>
_CCCL_DEVICE static inline bool mbarrier_test_wait(::cuda::std::uint64_t* __addr, const ::cuda::std::uint64_t& __state)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 800
::cuda::std::uint32_t __waitComplete;
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.test_wait.shared.b64 P_OUT, [%1], %2; // 1. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state)
: "memory");
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_80__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 700
/*
// mbarrier.test_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state; // 2. PTX
ISA 80, SM_90
// .sem = { .acquire }
// .scope = { .cta, .cluster }
template <cuda::ptx::dot_scope Scope>
__device__ static inline bool mbarrier_test_wait(
cuda::ptx::sem_acquire_t,
cuda::ptx::scope_t<Scope> scope,
uint64_t* addr,
const uint64_t& state);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline bool mbarrier_test_wait(
::cuda::ptx::sem_acquire_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint64_t& __state)
{
// __sem == sem_acquire (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
if constexpr (__scope == scope_cta)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.test_wait.acquire.cta.shared::cta.b64 P_OUT, [%1], %2; // 2. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.test_wait.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2; // 2. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state)
: "memory");
}
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// mbarrier.test_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state; // PTX ISA 86, SM_90
// .sem = { .relaxed }
// .scope = { .cta, .cluster }
template <cuda::ptx::dot_scope Scope>
__device__ static inline bool mbarrier_test_wait(
cuda::ptx::sem_relaxed_t,
cuda::ptx::scope_t<Scope> scope,
uint64_t* addr,
const uint64_t& state);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline bool mbarrier_test_wait(
::cuda::ptx::sem_relaxed_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint64_t& __state)
{
// __sem == sem_relaxed (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
if constexpr (__scope == scope_cta)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.test_wait.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2;\n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.test_wait.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2;\n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state)
: "memory");
}
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 860
#endif // _CUDA_PTX_GENERATED_MBARRIER_TEST_WAIT_H_

144
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/mbarrier_test_wait_parity.h
View File

@@ -1,144 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_MBARRIER_TEST_WAIT_PARITY_H_
#define _CUDA_PTX_GENERATED_MBARRIER_TEST_WAIT_PARITY_H_
/*
// mbarrier.test_wait.parity.shared.b64 waitComplete, [addr], phaseParity; // 3. PTX
ISA 71, SM_80 template <typename = void>
__device__ static inline bool mbarrier_test_wait_parity(
uint64_t* addr,
const uint32_t& phaseParity);
*/
#if __cccl_ptx_isa >= 710
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_80__();
template <typename = void>
_CCCL_DEVICE static inline bool
mbarrier_test_wait_parity(::cuda::std::uint64_t* __addr, const ::cuda::std::uint32_t& __phaseParity)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 800
::cuda::std::uint32_t __waitComplete;
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.test_wait.parity.shared.b64 P_OUT, [%1], %2; // 3. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
: "memory");
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_80__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 710
/*
// mbarrier.test_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity; // 4. PTX
ISA 80, SM_90
// .sem = { .acquire }
// .scope = { .cta, .cluster }
template <cuda::ptx::dot_scope Scope>
__device__ static inline bool mbarrier_test_wait_parity(
cuda::ptx::sem_acquire_t,
cuda::ptx::scope_t<Scope> scope,
uint64_t* addr,
const uint32_t& phaseParity);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline bool mbarrier_test_wait_parity(
::cuda::ptx::sem_acquire_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __phaseParity)
{
// __sem == sem_acquire (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
if constexpr (__scope == scope_cta)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.test_wait.parity.acquire.cta.shared::cta.b64 P_OUT, [%1], %2; // 4. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.test_wait.parity.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2; // 4. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
: "memory");
}
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// mbarrier.test_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity; // PTX ISA 86, SM_90
// .sem = { .relaxed }
// .scope = { .cta, .cluster }
template <cuda::ptx::dot_scope Scope>
__device__ static inline bool mbarrier_test_wait_parity(
cuda::ptx::sem_relaxed_t,
cuda::ptx::scope_t<Scope> scope,
uint64_t* addr,
const uint32_t& phaseParity);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline bool mbarrier_test_wait_parity(
::cuda::ptx::sem_relaxed_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __phaseParity)
{
// __sem == sem_relaxed (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
if constexpr (__scope == scope_cta)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.test_wait.parity.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2;\n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.test_wait.parity.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2;\n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
: "memory");
}
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 860
#endif // _CUDA_PTX_GENERATED_MBARRIER_TEST_WAIT_PARITY_H_

286
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/mbarrier_try_wait.h
View File

@@ -1,286 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_MBARRIER_TRY_WAIT_H_
#define _CUDA_PTX_GENERATED_MBARRIER_TRY_WAIT_H_
/*
// mbarrier.try_wait.shared::cta.b64 waitComplete, [addr], state; // 5a.
PTX ISA 78, SM_90 template <typename = void>
__device__ static inline bool mbarrier_try_wait(
uint64_t* addr,
const uint64_t& state);
*/
#if __cccl_ptx_isa >= 780
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline bool mbarrier_try_wait(::cuda::std::uint64_t* __addr, const ::cuda::std::uint64_t& __state)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.shared::cta.b64 P_OUT, [%1], %2; // 5a. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state)
: "memory");
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 780
/*
// mbarrier.try_wait.shared::cta.b64 waitComplete, [addr], state, suspendTimeHint; // 5b. PTX
ISA 78, SM_90 template <typename = void>
__device__ static inline bool mbarrier_try_wait(
uint64_t* addr,
const uint64_t& state,
const uint32_t& suspendTimeHint);
*/
#if __cccl_ptx_isa >= 780
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline bool mbarrier_try_wait(
::cuda::std::uint64_t* __addr, const ::cuda::std::uint64_t& __state, const ::cuda::std::uint32_t& __suspendTimeHint)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.shared::cta.b64 P_OUT, [%1], %2, %3; // 5b. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state), "r"(__suspendTimeHint)
: "memory");
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 780
/*
// mbarrier.try_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state; // 6a. PTX
ISA 80, SM_90
// .sem = { .acquire }
// .scope = { .cta, .cluster }
template <cuda::ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait(
cuda::ptx::sem_acquire_t,
cuda::ptx::scope_t<Scope> scope,
uint64_t* addr,
const uint64_t& state);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline bool mbarrier_try_wait(
::cuda::ptx::sem_acquire_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint64_t& __state)
{
// __sem == sem_acquire (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
if constexpr (__scope == scope_cta)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.acquire.cta.shared::cta.b64 P_OUT, [%1], %2; // 6a. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2; // 6a. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state)
: "memory");
}
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// mbarrier.try_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state , suspendTimeHint; // 6b. PTX
ISA 80, SM_90
// .sem = { .acquire }
// .scope = { .cta, .cluster }
template <cuda::ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait(
cuda::ptx::sem_acquire_t,
cuda::ptx::scope_t<Scope> scope,
uint64_t* addr,
const uint64_t& state,
const uint32_t& suspendTimeHint);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline bool mbarrier_try_wait(
::cuda::ptx::sem_acquire_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint64_t& __state,
const ::cuda::std::uint32_t& __suspendTimeHint)
{
// __sem == sem_acquire (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
if constexpr (__scope == scope_cta)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.acquire.cta.shared::cta.b64 P_OUT, [%1], %2 , %3; // 6b. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state), "r"(__suspendTimeHint)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2 , %3; // 6b. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state), "r"(__suspendTimeHint)
: "memory");
}
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// mbarrier.try_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state, suspendTimeHint; // PTX ISA 86, SM_90
// .sem = { .relaxed }
// .scope = { .cta, .cluster }
template <cuda::ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait(
cuda::ptx::sem_relaxed_t,
cuda::ptx::scope_t<Scope> scope,
uint64_t* addr,
const uint64_t& state,
const uint32_t& suspendTimeHint);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline bool mbarrier_try_wait(
::cuda::ptx::sem_relaxed_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint64_t& __state,
const ::cuda::std::uint32_t& __suspendTimeHint)
{
// __sem == sem_relaxed (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
if constexpr (__scope == scope_cta)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2, %3;\n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state), "r"(__suspendTimeHint)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2, %3;\n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state), "r"(__suspendTimeHint)
: "memory");
}
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// mbarrier.try_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state; // PTX ISA 86, SM_90
// .sem = { .relaxed }
// .scope = { .cta, .cluster }
template <cuda::ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait(
cuda::ptx::sem_relaxed_t,
cuda::ptx::scope_t<Scope> scope,
uint64_t* addr,
const uint64_t& state);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline bool mbarrier_try_wait(
::cuda::ptx::sem_relaxed_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint64_t& __state)
{
// __sem == sem_relaxed (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
if constexpr (__scope == scope_cta)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2;\n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2;\n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "l"(__state)
: "memory");
}
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 860
#endif // _CUDA_PTX_GENERATED_MBARRIER_TRY_WAIT_H_

290
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/mbarrier_try_wait_parity.h
View File

@@ -1,290 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_MBARRIER_TRY_WAIT_PARITY_H_
#define _CUDA_PTX_GENERATED_MBARRIER_TRY_WAIT_PARITY_H_
/*
// mbarrier.try_wait.parity.shared::cta.b64 waitComplete, [addr], phaseParity; // 7a.
PTX ISA 78, SM_90 template <typename = void>
__device__ static inline bool mbarrier_try_wait_parity(
uint64_t* addr,
const uint32_t& phaseParity);
*/
#if __cccl_ptx_isa >= 780
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline bool
mbarrier_try_wait_parity(::cuda::std::uint64_t* __addr, const ::cuda::std::uint32_t& __phaseParity)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.parity.shared::cta.b64 P_OUT, [%1], %2; // 7a. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
: "memory");
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 780
/*
// mbarrier.try_wait.parity.shared::cta.b64 waitComplete, [addr], phaseParity, suspendTimeHint; // 7b.
PTX ISA 78, SM_90 template <typename = void>
__device__ static inline bool mbarrier_try_wait_parity(
uint64_t* addr,
const uint32_t& phaseParity,
const uint32_t& suspendTimeHint);
*/
#if __cccl_ptx_isa >= 780
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
template <typename = void>
_CCCL_DEVICE static inline bool mbarrier_try_wait_parity(
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __phaseParity,
const ::cuda::std::uint32_t& __suspendTimeHint)
{
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.parity.shared::cta.b64 P_OUT, [%1], %2, %3; // 7b. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity), "r"(__suspendTimeHint)
: "memory");
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 780
/*
// mbarrier.try_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity; // 8a. PTX
ISA 80, SM_90
// .sem = { .acquire }
// .scope = { .cta, .cluster }
template <cuda::ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait_parity(
cuda::ptx::sem_acquire_t,
cuda::ptx::scope_t<Scope> scope,
uint64_t* addr,
const uint32_t& phaseParity);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline bool mbarrier_try_wait_parity(
::cuda::ptx::sem_acquire_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __phaseParity)
{
// __sem == sem_acquire (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
if constexpr (__scope == scope_cta)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.parity.acquire.cta.shared::cta.b64 P_OUT, [%1], %2; // 8a. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2; // 8a. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
: "memory");
}
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// mbarrier.try_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity, suspendTimeHint; // 8b. PTX
ISA 80, SM_90
// .sem = { .acquire }
// .scope = { .cta, .cluster }
template <cuda::ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait_parity(
cuda::ptx::sem_acquire_t,
cuda::ptx::scope_t<Scope> scope,
uint64_t* addr,
const uint32_t& phaseParity,
const uint32_t& suspendTimeHint);
*/
#if __cccl_ptx_isa >= 800
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline bool mbarrier_try_wait_parity(
::cuda::ptx::sem_acquire_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __phaseParity,
const ::cuda::std::uint32_t& __suspendTimeHint)
{
// __sem == sem_acquire (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
if constexpr (__scope == scope_cta)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.parity.acquire.cta.shared::cta.b64 P_OUT, [%1], %2, %3; // 8b. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity), "r"(__suspendTimeHint)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2, %3; // 8b. \n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity), "r"(__suspendTimeHint)
: "memory");
}
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 800
/*
// mbarrier.try_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity, suspendTimeHint; // PTX ISA 86,
SM_90
// .sem = { .relaxed }
// .scope = { .cta, .cluster }
template <cuda::ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait_parity(
cuda::ptx::sem_relaxed_t,
cuda::ptx::scope_t<Scope> scope,
uint64_t* addr,
const uint32_t& phaseParity,
const uint32_t& suspendTimeHint);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline bool mbarrier_try_wait_parity(
::cuda::ptx::sem_relaxed_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __phaseParity,
const ::cuda::std::uint32_t& __suspendTimeHint)
{
// __sem == sem_relaxed (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
if constexpr (__scope == scope_cta)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.parity.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2, %3;\n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity), "r"(__suspendTimeHint)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.parity.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2, %3;\n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity), "r"(__suspendTimeHint)
: "memory");
}
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 860
/*
// mbarrier.try_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity; // PTX ISA 86, SM_90
// .sem = { .relaxed }
// .scope = { .cta, .cluster }
template <cuda::ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait_parity(
cuda::ptx::sem_relaxed_t,
cuda::ptx::scope_t<Scope> scope,
uint64_t* addr,
const uint32_t& phaseParity);
*/
#if __cccl_ptx_isa >= 860
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
template <::cuda::ptx::dot_scope _Scope>
_CCCL_DEVICE static inline bool mbarrier_try_wait_parity(
::cuda::ptx::sem_relaxed_t,
::cuda::ptx::scope_t<_Scope> __scope,
::cuda::std::uint64_t* __addr,
const ::cuda::std::uint32_t& __phaseParity)
{
// __sem == sem_relaxed (due to parameter type constraint)
static_assert(__scope == scope_cta || __scope == scope_cluster);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
::cuda::std::uint32_t __waitComplete;
if constexpr (__scope == scope_cta)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.parity.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2;\n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
: "memory");
}
else if constexpr (__scope == scope_cluster)
{
asm("{\n\t .reg .pred P_OUT; \n\t"
"mbarrier.try_wait.parity.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2;\n\t"
"selp.b32 %0, 1, 0, P_OUT; \n"
"}"
: "=r"(__waitComplete)
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
: "memory");
}
return static_cast<bool>(__waitComplete);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
return false;
# endif
}
#endif // __cccl_ptx_isa >= 860
#endif // _CUDA_PTX_GENERATED_MBARRIER_TRY_WAIT_PARITY_H_

96
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/shl.h
View File

@@ -1,96 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_SHL_H_
#define _CUDA_PTX_GENERATED_SHL_H_
/*
// shl.b16 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename B16, enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 shl(
B16 a_reg,
uint32_t b_reg);
*/
#if __cccl_ptx_isa >= 100
extern "C" _CCCL_DEVICE void __cuda_ptx_shl_is_not_supported_before_SM_50__();
template <typename _B16, ::cuda::std::enable_if_t<sizeof(_B16) == 2, bool> = true>
_CCCL_DEVICE static inline _B16 shl(_B16 __a_reg, ::cuda::std::uint32_t __b_reg)
{
static_assert(sizeof(_B16) == 2);
static_assert(sizeof(_B16) == 2);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
::cuda::std::uint16_t __dest;
asm("shl.b16 %0, %1, %2;"
: "=h"(__dest)
: "h"(/*as_b16*/ *reinterpret_cast<const ::cuda::std::int16_t*>(&__a_reg)), "r"(__b_reg)
:);
return *reinterpret_cast<_B16*>(&__dest);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_shl_is_not_supported_before_SM_50__();
::cuda::std::uint16_t __err_out_var = 0;
return *reinterpret_cast<_B16*>(&__err_out_var);
# endif
}
#endif // __cccl_ptx_isa >= 100
/*
// shl.b32 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 shl(
B32 a_reg,
uint32_t b_reg);
*/
#if __cccl_ptx_isa >= 100
extern "C" _CCCL_DEVICE void __cuda_ptx_shl_is_not_supported_before_SM_50__();
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
_CCCL_DEVICE static inline _B32 shl(_B32 __a_reg, ::cuda::std::uint32_t __b_reg)
{
static_assert(sizeof(_B32) == 4);
static_assert(sizeof(_B32) == 4);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
::cuda::std::uint32_t __dest;
asm("shl.b32 %0, %1, %2;"
: "=r"(__dest)
: "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)), "r"(__b_reg)
:);
return *reinterpret_cast<_B32*>(&__dest);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_shl_is_not_supported_before_SM_50__();
::cuda::std::uint32_t __err_out_var = 0;
return *reinterpret_cast<_B32*>(&__err_out_var);
# endif
}
#endif // __cccl_ptx_isa >= 100
/*
// shl.b64 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename B64, enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 shl(
B64 a_reg,
uint32_t b_reg);
*/
#if __cccl_ptx_isa >= 100
extern "C" _CCCL_DEVICE void __cuda_ptx_shl_is_not_supported_before_SM_50__();
template <typename _B64, ::cuda::std::enable_if_t<sizeof(_B64) == 8, bool> = true>
_CCCL_DEVICE static inline _B64 shl(_B64 __a_reg, ::cuda::std::uint32_t __b_reg)
{
static_assert(sizeof(_B64) == 8);
static_assert(sizeof(_B64) == 8);
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
::cuda::std::uint64_t __dest;
asm("shl.b64 %0, %1, %2;"
: "=l"(__dest)
: "l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__a_reg)), "r"(__b_reg)
:);
return *reinterpret_cast<_B64*>(&__dest);
# else
// Unsupported architectures will have a linker error with a semi-decent error message
__cuda_ptx_shl_is_not_supported_before_SM_50__();
::cuda::std::uint64_t __err_out_var = 0;
return *reinterpret_cast<_B64*>(&__err_out_var);
# endif
}
#endif // __cccl_ptx_isa >= 100
#endif // _CUDA_PTX_GENERATED_SHL_H_

135
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/shr.h
View File

@@ -1,135 +0,0 @@
// This file was automatically generated. Do not edit.
#ifndef _CUDA_PTX_GENERATED_SHR_H_
#define _CUDA_PTX_GENERATED_SHR_H_
/*
// shr.b16 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename B16, enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 shr(
B16 a_reg,
uint32_t b_reg);
*/
#if __cccl_ptx_isa >= 100
template <typename _B16, ::cuda::std::enable_if_t<sizeof(_B16) == 2, bool> = true>
_CCCL_DEVICE static inline _B16 shr(_B16 __a_reg, ::cuda::std::uint32_t __b_reg)
{
static_assert(sizeof(_B16) == 2);
static_assert(sizeof(_B16) == 2);
::cuda::std::uint16_t __dest;
asm("shr.b16 %0, %1, %2;"
: "=h"(__dest)
: "h"(/*as_b16*/ *reinterpret_cast<const ::cuda::std::int16_t*>(&__a_reg)), "r"(__b_reg)
:);
return *reinterpret_cast<_B16*>(&__dest);
}
#endif // __cccl_ptx_isa >= 100
/*
// shr.b32 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename B32, enable_if_t<sizeof(B32) == 4 && !(is_integral_v<B32> && is_signed_v<B32>), bool> = true>
__device__ static inline B32 shr(
B32 a_reg,
uint32_t b_reg);
*/
#if __cccl_ptx_isa >= 100
template <
typename _B32,
::cuda::std::enable_if_t<sizeof(_B32) == 4 && !(::cuda::std::is_integral_v<_B32> && ::cuda::std::is_signed_v<_B32>),
bool> = true>
_CCCL_DEVICE static inline _B32 shr(_B32 __a_reg, ::cuda::std::uint32_t __b_reg)
{
::cuda::std::uint32_t __dest;
asm("shr.b32 %0, %1, %2;"
: "=r"(__dest)
: "r"(*reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)), "r"(__b_reg)
:);
return *reinterpret_cast<_B32*>(&__dest);
}
#endif // __cccl_ptx_isa >= 100
/*
// shr.b64 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename B64, enable_if_t<sizeof(B64) == 8 && !(is_integral_v<B64> && is_signed_v<B64>), bool> = true>
__device__ static inline B64 shr(
B64 a_reg,
uint32_t b_reg);
*/
#if __cccl_ptx_isa >= 100
template <
typename _B64,
::cuda::std::enable_if_t<sizeof(_B64) == 8 && !(::cuda::std::is_integral_v<_B64> && ::cuda::std::is_signed_v<_B64>),
bool> = true>
_CCCL_DEVICE static inline _B64 shr(_B64 __a_reg, ::cuda::std::uint32_t __b_reg)
{
::cuda::std::uint64_t __dest;
asm("shr.b64 %0, %1, %2;"
: "=l"(__dest)
: "l"(*reinterpret_cast<const ::cuda::std::int64_t*>(&__a_reg)), "r"(__b_reg)
:);
return *reinterpret_cast<_B64*>(&__dest);
}
#endif // __cccl_ptx_isa >= 100
/*
// shr.s16 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename = void>
__device__ static inline int16_t shr(
int16_t a_reg,
uint32_t b_reg);
*/
#if __cccl_ptx_isa >= 100
template <typename = void>
_CCCL_DEVICE static inline ::cuda::std::int16_t shr(::cuda::std::int16_t __a_reg, ::cuda::std::uint32_t __b_reg)
{
::cuda::std::int16_t __dest;
asm("shr.s16 %0, %1, %2;" : "=h"(__dest) : "h"(__a_reg), "r"(__b_reg) :);
return __dest;
}
#endif // __cccl_ptx_isa >= 100
/*
// shr.s32 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename S32, enable_if_t<sizeof(S32) == 4 && is_integral_v<S32> && is_signed_v<S32>, bool> = true>
__device__ static inline S32 shr(
S32 a_reg,
uint32_t b_reg);
*/
#if __cccl_ptx_isa >= 100
template <typename _S32,
::cuda::std::enable_if_t<sizeof(_S32) == 4 && ::cuda::std::is_integral_v<_S32>&& ::cuda::std::is_signed_v<_S32>,
bool> = true>
_CCCL_DEVICE static inline _S32 shr(_S32 __a_reg, ::cuda::std::uint32_t __b_reg)
{
::cuda::std::int32_t __dest;
asm("shr.s32 %0, %1, %2;"
: "=r"(__dest)
: "r"(*reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)), "r"(__b_reg)
:);
return *reinterpret_cast<_S32*>(&__dest);
}
#endif // __cccl_ptx_isa >= 100
/*
// shr.s64 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename S64, enable_if_t<sizeof(S64) == 8 && is_integral_v<S64> && is_signed_v<S64>, bool> = true>
__device__ static inline S64 shr(
S64 a_reg,
uint32_t b_reg);
*/
#if __cccl_ptx_isa >= 100
template <typename _S64,
::cuda::std::enable_if_t<sizeof(_S64) == 8 && ::cuda::std::is_integral_v<_S64>&& ::cuda::std::is_signed_v<_S64>,
bool> = true>
_CCCL_DEVICE static inline _S64 shr(_S64 __a_reg, ::cuda::std::uint32_t __b_reg)
{
::cuda::std::int64_t __dest;
asm("shr.s64 %0, %1, %2;"
: "=l"(__dest)
: "l"(*reinterpret_cast<const ::cuda::std::int64_t*>(&__a_reg)), "r"(__b_reg)
:);
return *reinterpret_cast<_S64*>(&__dest);
}
#endif // __cccl_ptx_isa >= 100
#endif // _CUDA_PTX_GENERATED_SHR_H_

45
qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/mbarrier_arrive.h
View File

@@ -1,45 +0,0 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA_PTX_MBARRIER_ARRIVE_H_
#define _CUDA_PTX_MBARRIER_ARRIVE_H_
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__ptx/ptx_dot_variants.h>
#include <cuda/__ptx/ptx_helper_functions.h>
#include <cuda/std/cstdint>
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
// 9.7.12.15.13. Parallel Synchronization and Communication Instructions: mbarrier.arrive
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive
#include <cuda/__ptx/instructions/generated/mbarrier_arrive.h>
#include <cuda/__ptx/instructions/generated/mbarrier_arrive_expect_tx.h>
#include <cuda/__ptx/instructions/generated/mbarrier_arrive_no_complete.h>
_CCCL_END_NAMESPACE_CUDA_PTX
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA_PTX_MBARRIER_ARRIVE_H_

Some files were not shown because too many files have changed in this diff Show More