dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

Browse Source 
CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0
This commit is contained in:
EngineX CI
2026-07-30 09:35:51 +00:00
parent b4d01f481e
commit 56fd68e7dd
8871 changed files with 1454674 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

394
cccl_upstream/libcudacxx/include/cuda/__numeric/add_overflow.h Normal file
View File

@@ -0,0 +1,394 @@
//===----------------------------------------------------------------------===//
//
// Part of the libcu++ Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___NUMERIC_ADD_OVERFLOW_H
#define _CUDA___NUMERIC_ADD_OVERFLOW_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__cmath/uabs.h>
#include <cuda/__numeric/overflow_cast.h>
#include <cuda/__numeric/overflow_result.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__type_traits/common_type.h>
#include <cuda/std/__type_traits/conditional.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__type_traits/is_unsigned.h>
#include <cuda/std/__type_traits/is_void.h>
#include <cuda/std/__type_traits/make_nbit_int.h>
#include <cuda/std/__type_traits/make_signed.h>
#include <cuda/std/__type_traits/make_unsigned.h>
#include <cuda/std/__type_traits/num_bits.h>
#include <cuda/std/__utility/cmp.h>
#include <cuda/std/cstdint>
#include <nv/target>
#if _CCCL_COMPILER(MSVC) && _CCCL_HOST_ARCH(X86_64)
# include <intrin.h>
#endif // _CCCL_COMPILER(MSVC) && _CCCL_HOST_ARCH(X86_64)
#include <cuda/std/__cccl/prologue.h>
#if _CCCL_CHECK_BUILTIN(builtin_add_overflow) || _CCCL_COMPILER(GCC)
# define _CCCL_BUILTIN_ADD_OVERFLOW(...) __builtin_add_overflow(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(builtin_add_overflow)
// nvc++ < 26.1 doesn't support 128-bit integers and crashes when certain type combinations are used (nvbug 5730860).
#if _CCCL_COMPILER(NVHPC, <, 26, 1)
# undef _CCCL_BUILTIN_ADD_OVERFLOW
#endif // _CCCL_COMPILER(NVHPC, <, 26, 1)
_CCCL_BEGIN_NAMESPACE_CUDA
template <class _Tp>
[[nodiscard]] _CCCL_API constexpr overflow_result<_Tp> __add_overflow_generic_impl(_Tp __lhs, _Tp __rhs) noexcept
{
using _Up = ::cuda::std::make_unsigned_t<_Tp>;
auto __sum = static_cast<_Tp>(static_cast<_Up>(__lhs) + static_cast<_Up>(__rhs));
if constexpr (::cuda::std::is_signed_v<_Tp>)
{
return {__sum, (__sum < __lhs) == (__rhs >= _Tp{0})};
}
else
{
return {__sum, __sum < __lhs};
}
}
#if _CCCL_DEVICE_COMPILATION()
template <class _Tp>
[[nodiscard]] _CCCL_DEVICE_API overflow_result<_Tp> __add_overflow_device(_Tp __lhs, _Tp __rhs) noexcept
{
if constexpr (::cuda::std::is_unsigned_v<_Tp>)
{
using ::cuda::std::uint32_t;
using ::cuda::std::uint64_t;
if constexpr (sizeof(_Tp) < sizeof(uint32_t))
{
const auto __result = uint32_t{__lhs} + uint32_t{__rhs};
return {static_cast<_Tp>(__result), !::cuda::std::in_range<_Tp>(__result)};
}
else if constexpr (sizeof(_Tp) == sizeof(uint32_t))
{
uint32_t __result;
int __overflow;
asm("add.cc.u32 %0, %2, %3;"
"addc.u32 %1, 0, 0;"
: "=r"(__result), "=r"(__overflow)
: "r"(__lhs), "r"(__rhs));
return {__result, static_cast<bool>(__overflow)};
}
else if constexpr (sizeof(_Tp) == sizeof(uint64_t))
{
uint64_t __result;
int __overflow;
asm("add.cc.u64 %0, %2, %3;"
"addc.u32 %1, 0, 0;"
: "=l"(__result), "=r"(__overflow)
: "l"(__lhs), "l"(__rhs));
return {__result, static_cast<bool>(__overflow)};
}
# if _CCCL_HAS_INT128()
else if constexpr (sizeof(_Tp) == sizeof(__uint128_t))
{
uint64_t __result_lo;
uint64_t __result_hi;
int __overflow;
asm("add.cc.u64 %1, %4, %6;"
"addc.cc.u64 %0, %3, %5;"
"addc.u32 %2, 0, 0;"
: "=l"(__result_hi), "=l"(__result_lo), "=r"(__overflow)
: "l"(static_cast<uint64_t>(__lhs >> 64)),
"l"(static_cast<uint64_t>(__lhs)),
"l"(static_cast<uint64_t>(__rhs >> 64)),
"l"(static_cast<uint64_t>(__rhs)));
return {(static_cast<__uint128_t>(__result_hi) << 64) | __result_lo, static_cast<bool>(__overflow)};
}
# endif // _CCCL_HAS_INT128()
else
{
::cuda::__add_overflow_generic_impl(__lhs, __rhs); // do not use builtin functions
}
}
else
{
using ::cuda::std::int32_t;
if constexpr (sizeof(_Tp) < sizeof(int32_t))
{
const auto __result = int32_t{__lhs} + int32_t{__rhs};
return {static_cast<_Tp>(__result), !::cuda::std::in_range<_Tp>(__result)};
}
# if _CCCL_HAS_INT128()
else if constexpr (sizeof(_Tp) == sizeof(__int128_t))
{
using _Up = ::cuda::std::make_unsigned_t<_Tp>;
const auto __uadd_result = ::cuda::__add_overflow_device(static_cast<_Up>(__lhs), static_cast<_Up>(__rhs));
const auto __result = static_cast<_Tp>(__uadd_result.value);
const auto __overflow = ((__lhs >= 0) == (__rhs >= 0)) && (__uadd_result.overflow == (__result >= 0));
return {__result, __overflow};
}
# endif // _CCCL_HAS_INT128()
else
{
// For 32 and 64 bit ints, this seems to be the more efficient path.
return ::cuda::__add_overflow_generic_impl(__lhs, __rhs);
}
}
}
#endif // _CCCL_DEVICE_COMPILATION()
#if _CCCL_HOST_COMPILATION()
template <class _Tp>
[[nodiscard]] _CCCL_HOST_API overflow_result<_Tp> __add_overflow_host(_Tp __lhs, _Tp __rhs) noexcept
{
# if _CCCL_COMPILER(MSVC) && _CCCL_HOST_ARCH(X86_64)
if constexpr (sizeof(_Tp) <= 8)
{
# if _CCCL_COMPILER(MSVC, >=, 19, 37)
if constexpr (::cuda::std::is_signed_v<_Tp>)
{
overflow_result<_Tp> __result;
if constexpr (sizeof(_Tp) == 1)
{
__result.overflow = ::_add_overflow_i8(0, __lhs, __rhs, &__result.value);
}
else if constexpr (sizeof(_Tp) == 2)
{
__result.overflow = ::_add_overflow_i16(0, __lhs, __rhs, &__result.value);
}
else if constexpr (sizeof(_Tp) == 4)
{
__result.overflow = ::_add_overflow_i32(0, __lhs, __rhs, &__result.value);
}
else if constexpr (sizeof(_Tp) == 8)
{
__result.overflow = ::_add_overflow_i64(0, __lhs, __rhs, &__result.value);
}
return __result;
}
else
# endif // _CCCL_COMPILER(MSVC, >=, 19, 37)
if constexpr (::cuda::std::is_unsigned_v<_Tp>)
{ // unsigned
overflow_result<_Tp> __result;
if constexpr (sizeof(_Tp) == 1)
{
__result.overflow = ::_addcarry_u8(0, __lhs, __rhs, &__result.value);
}
else if constexpr (sizeof(_Tp) == 2)
{
__result.overflow = ::_addcarry_u16(0, __lhs, __rhs, &__result.value);
}
else if constexpr (sizeof(_Tp) == 4)
{
__result.overflow = ::_addcarry_u32(0, __lhs, __rhs, &__result.value);
}
else if constexpr (sizeof(_Tp) == 8)
{
__result.overflow = ::_addcarry_u64(0, __lhs, __rhs, &__result.value);
}
return __result;
}
else
{
return ::cuda::__add_overflow_generic_impl(__lhs, __rhs);
}
}
else
# endif // ^^^ _CCCL_COMPILER(MSVC) || _CCCL_HOST_ARCH(X86_64) ^^^
{
return ::cuda::__add_overflow_generic_impl(__lhs, __rhs);
}
}
#endif // _CCCL_HOST_COMPILATION()
template <typename _Tp>
[[nodiscard]] _CCCL_API constexpr overflow_result<_Tp> __add_overflow_uniform_type(_Tp __lhs, _Tp __rhs) noexcept
{
#if !_CCCL_TILE_COMPILATION() // error: asm statement is unsupported in tile code
_CCCL_IF_NOT_CONSTEVAL_DEFAULT
{
NV_IF_TARGET(NV_IS_DEVICE,
(return ::cuda::__add_overflow_device(__lhs, __rhs);),
(return ::cuda::__add_overflow_host(__lhs, __rhs);))
}
#endif // !_CCCL_TILE_COMPILATION()
return ::cuda::__add_overflow_generic_impl(__lhs, __rhs);
}
template <typename _Result, typename _Lhs, typename _Rhs>
inline constexpr bool __is_add_representable_v =
sizeof(_Result) > sizeof(_Lhs) && sizeof(_Result) > sizeof(_Rhs)
&& (::cuda::std::is_signed_v<_Result>
|| (::cuda::std::is_unsigned_v<_Lhs> && ::cuda::std::is_unsigned_v<_Rhs> && ::cuda::std::is_unsigned_v<_Result>) );
/***********************************************************************************************************************
* Public interface
**********************************************************************************************************************/
_CCCL_TEMPLATE(typename _Result = void,
typename _Lhs,
typename _Rhs,
typename _Common = ::cuda::std::common_type_t<_Lhs, _Rhs>,
typename _ActualResult = ::cuda::std::conditional_t<::cuda::std::is_void_v<_Result>, _Common, _Result>)
_CCCL_REQUIRES((::cuda::std::is_void_v<_Result> || ::cuda::std::__cccl_is_integer_v<_Result>)
_CCCL_AND ::cuda::std::__cccl_is_integer_v<_Lhs> _CCCL_AND ::cuda::std::__cccl_is_integer_v<_Rhs>)
[[nodiscard]]
_CCCL_API constexpr overflow_result<_ActualResult> add_overflow(const _Lhs __lhs, const _Rhs __rhs) noexcept
{
using ::cuda::std::is_same_v;
// We want to use __builtin_add_overflow only in host code. When compiling CUDA source file, we cannot use it in
// constant expressions, because it doesn't work before nvcc 13.1 and is buggy in 13.1. When compiling C++ source
// file, we can use it all the time.
#if defined(_CCCL_BUILTIN_ADD_OVERFLOW)
# if _CCCL_CUDA_COMPILATION()
_CCCL_IF_NOT_CONSTEVAL_DEFAULT
# endif // _CCCL_CUDA_COMPILATION()
{
// nvc++ doesn't support overflow builtins for 128-bit integers of different signedness.
# if _CCCL_COMPILER(NVHPC)
if constexpr ((sizeof(_ActualResult) != 16 && sizeof(_Lhs) != 16 && sizeof(_Rhs) != 16)
|| (is_same_v<_ActualResult, _Lhs> && is_same_v<_ActualResult, _Rhs>) )
# endif // _CCCL_COMPILER(NVHPC)
{
NV_IF_TARGET(NV_IS_HOST, ({
overflow_result<_ActualResult> __result{};
__result.overflow = _CCCL_BUILTIN_ADD_OVERFLOW(__lhs, __rhs, &__result.value);
return __result;
}))
}
}
#endif // _CCCL_BUILTIN_ADD_OVERFLOW
// Host fallback + device implementation.
#if _CCCL_CUDA_COMPILATION() || !defined(_CCCL_BUILTIN_ADD_OVERFLOW) || (_CCCL_COMPILER(NVHPC) && _CCCL_HAS_INT128())
using ::cuda::std::__make_nbit_int_t;
using ::cuda::std::__make_nbit_uint_t;
using ::cuda::std::__num_bits_v;
using ::cuda::std::is_signed_v;
using ::cuda::std::is_unsigned_v;
using _CommonAll = ::cuda::std::common_type_t<_Common, _ActualResult>;
[[maybe_unused]] const bool __is_lhs_ge_zero = is_unsigned_v<_Lhs> || __lhs >= 0;
[[maybe_unused]] const bool __is_rhs_ge_zero = is_unsigned_v<_Rhs> || __rhs >= 0;
// shortcut for the case where inputs are representable with the max type
if constexpr (__is_add_representable_v<_ActualResult, _Lhs, _Rhs>)
{
const auto __lhs1 = static_cast<_CommonAll>(__lhs);
const auto __rhs1 = static_cast<_CommonAll>(__rhs);
const auto __sum = static_cast<_CommonAll>(__lhs1 + __rhs1);
return ::cuda::overflow_cast<_ActualResult>(__sum);
}
// * int + int -> int
else if constexpr (is_signed_v<_Lhs> && is_signed_v<_Rhs> && is_signed_v<_ActualResult>) // all signed
{
using _Sp = __make_nbit_int_t<__num_bits_v<_CommonAll>>;
const auto __lhs1 = static_cast<_Sp>(__lhs);
const auto __rhs1 = static_cast<_Sp>(__rhs);
const auto __sum = ::cuda::__add_overflow_uniform_type(__lhs1, __rhs1);
const auto __ret = ::cuda::overflow_cast<_ActualResult>(__sum.value);
return overflow_result<_ActualResult>{__ret.value, __ret.overflow || __sum.overflow};
}
// Positive inputs
// * unsigned + unsigned (compile-time)
// * unsigned + int >= 0 (compile-time + run-time check)
// * int >= 0 + unsigned (compile-time + run-time check)
// * int >= 0 + int >= 0 -> _ActualResult=unsigned (_ActualResult=signed already handled above) (run-time check)
else if (__is_lhs_ge_zero && __is_rhs_ge_zero)
{
using _Up = __make_nbit_uint_t<__num_bits_v<_CommonAll>>;
const auto __lhs1 = static_cast<_Up>(__lhs);
const auto __rhs1 = static_cast<_Up>(__rhs);
const auto __sum = ::cuda::__add_overflow_uniform_type(__lhs1, __rhs1);
const auto __ret = ::cuda::overflow_cast<_ActualResult>(__sum.value);
return overflow_result<_ActualResult>{__ret.value, __ret.overflow || __sum.overflow};
}
// Negative inputs
// * int < 0 + int < 0 -> _ActualResult=unsigned (_ActualResult=signed already handled above) (run-time check)
else if (!__is_lhs_ge_zero && !__is_rhs_ge_zero)
{
const auto __lhs1 = static_cast<_ActualResult>(__lhs);
const auto __rhs1 = static_cast<_ActualResult>(__rhs);
return overflow_result<_ActualResult>{static_cast<_ActualResult>(__lhs1 + __rhs1), true};
}
// Opposite signs
// * int < 0 + int >= 0 -> _ActualResult=unsigned (_ActualResult=signed already handled above)
// * int >= 0 + int < 0 -> _ActualResult=unsigned (_ActualResult=signed already handled above)
else if constexpr (is_signed_v<_Lhs> && is_signed_v<_Rhs>)
{
return ::cuda::overflow_cast<_ActualResult>(static_cast<_Common>(__lhs) + static_cast<_Common>(__rhs));
}
// Opposite signs
// * unsigned + int < 0
// * int < 0 + unsigned
else
{
// skip checks in cmp_less, cmp_greater, uabs
if constexpr (is_unsigned_v<_Lhs> && is_signed_v<_Rhs>)
{
_CCCL_ASSUME(__rhs < 0);
}
if constexpr (is_unsigned_v<_Rhs> && is_signed_v<_Lhs>)
{
_CCCL_ASSUME(__lhs < 0);
}
const auto __lhs1 = static_cast<_CommonAll>(__lhs);
const auto __rhs1 = static_cast<_CommonAll>(__rhs);
const auto __sum = static_cast<_CommonAll>(__lhs1 + __rhs1); // no overflow because of opposite signs
// check if lhs + rhs is < 0, e.g. lhs >= 0 && lhs < |rhs|
if ((is_unsigned_v<_Lhs> && ::cuda::std::cmp_less(__lhs, ::cuda::uabs(__rhs)))
|| (is_unsigned_v<_Rhs> && ::cuda::std::cmp_greater(::cuda::uabs(__lhs), __rhs)))
{
if constexpr (is_unsigned_v<_ActualResult>)
{
return overflow_result<_ActualResult>{static_cast<_ActualResult>(__sum), true};
}
else
{
using _Sp = ::cuda::std::make_signed_t<_Common>;
return ::cuda::overflow_cast<_ActualResult>(static_cast<_Sp>(__sum));
}
}
return overflow_result<_ActualResult>{static_cast<_ActualResult>(__sum), false}; // because of opposite signs
}
#endif // _CCCL_CUDA_COMPILATION() || !_CCCL_BUILTIN_ADD_OVERFLOW || (_CCCL_COMPILER(NVHPC) && _CCCL_HAS_INT128())
}
//! @brief Adds two numbers \p __lhs and \p __rhs with overflow detection
_CCCL_TEMPLATE(typename _Result, typename _Lhs, typename _Rhs)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Result> _CCCL_AND ::cuda::std::__cccl_is_integer_v<_Lhs>
_CCCL_AND ::cuda::std::__cccl_is_integer_v<_Rhs>)
[[nodiscard]] _CCCL_API constexpr bool add_overflow(_Result& __result, const _Lhs __lhs, const _Rhs __rhs) noexcept
{
const auto __res = ::cuda::add_overflow<_Result>(__lhs, __rhs);
__result = __res.value;
return __res.overflow;
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___NUMERIC_ADD_OVERFLOW_H

150
cccl_upstream/libcudacxx/include/cuda/__numeric/div_overflow.h Normal file
View File

@@ -0,0 +1,150 @@
//===----------------------------------------------------------------------===//
//
// Part of the libcu++ Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___NUMERIC_DIV_OVERFLOW_H
#define _CUDA___NUMERIC_DIV_OVERFLOW_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__cmath/neg.h>
#include <cuda/__cmath/uabs.h>
#include <cuda/__numeric/overflow_cast.h>
#include <cuda/__numeric/overflow_result.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/common_type.h>
#include <cuda/std/__type_traits/conditional.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__type_traits/is_unsigned.h>
#include <cuda/std/__type_traits/is_void.h>
#include <cuda/std/__type_traits/make_unsigned.h>
#include <cuda/std/__utility/cmp.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <typename _Result, typename _Lhs, typename _Rhs>
inline constexpr bool __is_div_representable_v =
(sizeof(_Result) > sizeof(_Lhs) && sizeof(_Result) > sizeof(_Rhs) && ::cuda::std::is_signed_v<_Result>)
|| (sizeof(_Result) >= sizeof(_Lhs) && sizeof(_Result) >= sizeof(_Rhs)
&& ::cuda::std::is_unsigned_v<_Lhs> && ::cuda::std::is_unsigned_v<_Rhs> && ::cuda::std::is_unsigned_v<_Result>);
/***********************************************************************************************************************
* Public interface
**********************************************************************************************************************/
// div_overflow strategy:
//
// * If the result is representable with the actual result type, directly return the result
// * Both inputs are signed: check for min / -1, otherwise directly return the result
// * Both inputs are positive: directly return the result
// * Mixed signed/unsigned:
// - If the result is unsigned, return overflow (check for lhs != 0)
// - If the result is signed, compute |lhs| / |rhs| and return the result as negative (always representable)
_CCCL_TEMPLATE(typename _Result = void,
typename _Lhs,
typename _Rhs,
typename _Common = ::cuda::std::common_type_t<_Lhs, _Rhs>,
typename _ActualResult = ::cuda::std::conditional_t<::cuda::std::is_void_v<_Result>, _Common, _Result>)
_CCCL_REQUIRES((::cuda::std::is_void_v<_Result> || ::cuda::std::__cccl_is_integer_v<_Result>)
_CCCL_AND ::cuda::std::__cccl_is_integer_v<_Lhs> _CCCL_AND ::cuda::std::__cccl_is_integer_v<_Rhs>)
[[nodiscard]]
_CCCL_API constexpr overflow_result<_ActualResult> div_overflow(const _Lhs __lhs, const _Rhs __rhs) noexcept
{
_CCCL_ASSERT(__rhs != _Rhs{0}, "division by zero");
// the result is representable with the actual result type
if constexpr (__is_div_representable_v<_ActualResult, _Lhs, _Rhs>)
{
const auto __lhs1 = static_cast<_ActualResult>(__lhs);
const auto __rhs1 = static_cast<_ActualResult>(__rhs);
const auto __result = static_cast<_ActualResult>(__lhs1 / __rhs1);
return overflow_result<_ActualResult>{__result, false};
}
else
{
using ::cuda::std::is_signed_v;
using ::cuda::std::is_unsigned_v;
using ::cuda::std::make_unsigned_t;
using ::cuda::std::numeric_limits;
constexpr bool __both_signed = is_signed_v<_Lhs> && is_signed_v<_Rhs>;
[[maybe_unused]] const bool __lhs_ge_zero = is_unsigned_v<_Lhs> || __lhs >= _Lhs{0};
[[maybe_unused]] const bool __rhs_ge_zero = is_unsigned_v<_Rhs> || __rhs >= _Rhs{0};
if constexpr (__both_signed)
{
constexpr auto __lhs_min = numeric_limits<_Lhs>::min();
// special case for min / -1 -> potential overflow
if (__lhs == __lhs_min && __rhs == _Rhs{-1})
{
constexpr auto __neg_lhs_min = ::cuda::uabs(__lhs_min);
constexpr auto __result_max = numeric_limits<_ActualResult>::max();
const bool __overflow = ::cuda::std::cmp_greater(__neg_lhs_min, __result_max);
return overflow_result<_ActualResult>{static_cast<_ActualResult>(__neg_lhs_min), __overflow};
}
const auto __lhs1 = static_cast<_Common>(__lhs);
const auto __rhs1 = static_cast<_Common>(__rhs);
return ::cuda::overflow_cast<_ActualResult>(__lhs1 / __rhs1);
}
else if (__lhs_ge_zero && __rhs_ge_zero) // lhs and rhs are both >= 0
{
constexpr auto __result_max = numeric_limits<_ActualResult>::max();
using _UnsignedCommon = make_unsigned_t<_Common>;
const auto __lhs1 = static_cast<_UnsignedCommon>(__lhs);
const auto __rhs1 = static_cast<_UnsignedCommon>(__rhs);
const auto __result = __lhs1 / __rhs1;
const auto __is_overflow = ::cuda::std::cmp_greater(__result, __result_max);
return overflow_result<_ActualResult>{static_cast<_ActualResult>(__result), __is_overflow};
}
else // lhs and rhs are mixed positive/negative -> negative result
{
const auto __lhs1 = ::cuda::uabs(__lhs);
const auto __rhs1 = ::cuda::uabs(__rhs);
const auto __div = __lhs1 / __rhs1;
const auto __result = static_cast<_ActualResult>(::cuda::neg(__div));
if constexpr (is_unsigned_v<_ActualResult>)
{
return overflow_result<_ActualResult>{__result, __lhs != 0};
}
else
{
constexpr auto __result_min = numeric_limits<_ActualResult>::min();
constexpr auto __neg_result_min = ::cuda::uabs(__result_min);
const auto __is_overflow = ::cuda::std::cmp_greater(__div, __neg_result_min);
return overflow_result<_ActualResult>{__result, __is_overflow};
}
}
}
}
//! @brief Divides two numbers \p __lhs and \p __rhs with overflow detection
_CCCL_TEMPLATE(typename _Result, typename _Lhs, typename _Rhs)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Result> _CCCL_AND ::cuda::std::__cccl_is_integer_v<_Lhs>
_CCCL_AND ::cuda::std::__cccl_is_integer_v<_Rhs>)
[[nodiscard]] _CCCL_API constexpr bool div_overflow(_Result& __result, const _Lhs __lhs, const _Rhs __rhs) noexcept
{
const auto __res = ::cuda::div_overflow<_Result>(__lhs, __rhs);
__result = __res.value;
return __res.overflow;
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___NUMERIC_DIV_OVERFLOW_H

336
cccl_upstream/libcudacxx/include/cuda/__numeric/isclose.h Normal file
View File

@@ -0,0 +1,336 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___NUMERIC_ISCLOSE_H
#define _CUDA___NUMERIC_ISCLOSE_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__cmath/ceil_div.h>
#include <cuda/__cmath/mul_hi.h>
#include <cuda/__cmath/uabs.h>
#include <cuda/__complex/get_real_imag.h>
#include <cuda/__complex/traits.h>
#include <cuda/__type_traits/is_floating_point.h>
#include <cuda/__utility/in_range.h>
#include <cuda/std/__algorithm/max.h>
#include <cuda/std/__cmath/abs.h>
#include <cuda/std/__cmath/exponential_functions.h>
#include <cuda/std/__cmath/hypot.h>
#include <cuda/std/__cmath/isfinite.h>
#include <cuda/std/__cmath/min_max.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/conditional.h>
#include <cuda/std/__type_traits/is_extended_floating_point.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_same.h>
#include <cuda/std/__type_traits/is_unsigned.h>
#include <cuda/std/__type_traits/make_unsigned.h>
#include <cuda/std/__utility/cmp.h>
#include <cuda/std/cstdint>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <typename _Tp>
using __isclose_compare_t _CCCL_NODEBUG_ALIAS = ::cuda::std::
conditional_t<(::cuda::std::__is_extended_floating_point_v<_Tp> && sizeof(_Tp) <= sizeof(float)), float, _Tp>;
// compute 10^-(digits10 / 2)
template <typename _Tp>
[[nodiscard]] _CCCL_API _CCCL_CONSTEVAL float __isclose_default_relative_tolerance() noexcept
{
constexpr auto __digits = ::cuda::ceil_div(::cuda::std::numeric_limits<_Tp>::max_digits10, 2);
auto __exp = 1.0f;
for (int __i = 0; __i < __digits; ++__i)
{
__exp *= 10.0f;
}
return 1.0f / __exp;
}
template <typename _Tp>
[[nodiscard]] _CCCL_API constexpr bool
__isclose_fp_impl(const _Tp __lhs, const _Tp __rhs, const float __rel_tol, const _Tp __abs_tol) noexcept
{
_CCCL_ASSERT(::cuda::in_range(__rel_tol, 0.0f, 1.0f),
"cuda::isclose: relative tolerance must be in the range [0.0, 1.0]");
_CCCL_ASSERT(::cuda::std::isfinite(__abs_tol) && __abs_tol >= _Tp{0},
"cuda::isclose: absolute tolerance must be finite and non-negative");
if (__lhs == __rhs)
{
return true;
}
if (!::cuda::std::isfinite(__lhs) || !::cuda::std::isfinite(__rhs))
{
return false;
}
const auto __diff = ::cuda::std::fabs(__lhs - __rhs);
const auto __lhs_abs = ::cuda::std::fabs(__lhs);
const auto __rhs_abs = ::cuda::std::fabs(__rhs);
const auto __rel_value = static_cast<_Tp>(__rel_tol * ::cuda::std::fmax(__lhs_abs, __rhs_abs));
return __diff <= ::cuda::std::fmax(__abs_tol, __rel_value);
}
template <typename _ComplexType, typename _AbsTol>
[[nodiscard]] _CCCL_HOST_DEVICE_API bool __isclose_complex_impl(
const _ComplexType& __lhs, const _ComplexType& __rhs, const float __rel_tol, const _AbsTol __abs_tol) noexcept
{
using __scalar_t _CCCL_NODEBUG_ALIAS = typename _ComplexType::value_type;
using __compare_t _CCCL_NODEBUG_ALIAS = __isclose_compare_t<__scalar_t>;
static_assert(::cuda::is_floating_point_v<__scalar_t>, "cuda::isclose: __scalar_t must be a floating point type");
#if _CCCL_HAS_FLOAT128()
// __float128 is not supported because cuda::std::hypot is not implemented for this type
static_assert(!::cuda::std::is_same_v<__scalar_t, __float128>, "cuda::isclose: __float128 is not supported");
#endif // _CCCL_HAS_FLOAT128()
_CCCL_ASSERT(::cuda::in_range(__rel_tol, 0.0f, 1.0f),
"cuda::isclose: relative tolerance must be in the range [0.0, 1.0]");
_CCCL_ASSERT(::cuda::std::isfinite(__abs_tol) && __abs_tol >= __scalar_t{0},
"cuda::isclose: absolute tolerance must be finite and non-negative");
const auto __lhs_real = static_cast<__compare_t>(::cuda::__get_real(__lhs));
const auto __lhs_imag = static_cast<__compare_t>(::cuda::__get_imag(__lhs));
const auto __rhs_real = static_cast<__compare_t>(::cuda::__get_real(__rhs));
const auto __rhs_imag = static_cast<__compare_t>(::cuda::__get_imag(__rhs));
const auto __abs = static_cast<__compare_t>(__abs_tol);
if (__lhs_real == __rhs_real && __lhs_imag == __rhs_imag)
{
return true;
}
if (!::cuda::std::isfinite(__lhs_real) || !::cuda::std::isfinite(__lhs_imag) || !::cuda::std::isfinite(__rhs_real)
|| !::cuda::std::isfinite(__rhs_imag))
{
return false;
}
const auto __diff = ::cuda::std::hypot(__lhs_real - __rhs_real, __lhs_imag - __rhs_imag);
const auto __lhs_abs = ::cuda::std::hypot(__lhs_real, __lhs_imag);
const auto __rhs_abs = ::cuda::std::hypot(__rhs_real, __rhs_imag);
const auto __rel_value = __rel_tol * ::cuda::std::fmax(__lhs_abs, __rhs_abs);
return __diff <= ::cuda::std::fmax(__abs, __rel_value);
}
_CCCL_TEMPLATE(typename _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr ::cuda::std::make_unsigned_t<_Tp>
__safe_abs_diff(const _Tp __lhs, const _Tp __rhs) noexcept
{
using __unsigned_t _CCCL_NODEBUG_ALIAS = ::cuda::std::make_unsigned_t<_Tp>;
const auto __lhs_abs = ::cuda::uabs(__lhs);
const auto __rhs_abs = ::cuda::uabs(__rhs);
const auto __is_lhs_negative = ::cuda::std::cmp_less(__lhs, _Tp{0});
const auto __is_rhs_negative = ::cuda::std::cmp_less(__rhs, _Tp{0});
if (__is_lhs_negative != __is_rhs_negative)
{
return static_cast<__unsigned_t>(__lhs_abs + __rhs_abs);
}
return (__lhs_abs < __rhs_abs)
? static_cast<__unsigned_t>(__rhs_abs - __lhs_abs)
: static_cast<__unsigned_t>(__lhs_abs - __rhs_abs);
}
// Represents a non-negative float exactly as __mantissa_ / 2^__shift_.
struct __float_ratio
{
::cuda::std::uint32_t __mantissa_{};
int __shift_{};
_CCCL_HOST_DEVICE_API explicit __float_ratio(const float __value) noexcept
{
_CCCL_ASSERT(__value >= 0.0f, "cuda::__float_ratio: value must be non-negative");
constexpr int __digits = ::cuda::std::numeric_limits<float>::digits;
int __exponent = 0;
const auto __fraction = ::cuda::std::frexp(__value, &__exponent);
__mantissa_ = static_cast<::cuda::std::uint32_t>(::cuda::std::ldexp(__fraction, __digits));
__shift_ = __digits - __exponent;
}
template <typename _Unsigned>
[[nodiscard]] _CCCL_HOST_DEVICE_API _Unsigned operator*(const _Unsigned __value) const noexcept
{
static_assert(::cuda::std::is_unsigned_v<_Unsigned>, "cuda::__float_ratio::operator* requires an unsigned type");
// The result is floor(__value * __mantissa_ / 2^__shift_).
constexpr int __digits = ::cuda::std::numeric_limits<_Unsigned>::digits;
constexpr int __float_digits = ::cuda::std::numeric_limits<float>::digits;
constexpr auto __power_of_two_mant = ::cuda::std::uint32_t{1} << (__float_digits - 1);
static_assert(__digits >= __float_digits, "__float_ratio requires an unsigned integer at least as wide as float");
// A zero mantissa represents zero. If the shift is at least the width of the double-width product, all bits are
// shifted out and the result rounds down to zero.
if (__mantissa_ == 0 || __shift_ >= 2 * __digits)
{
return _Unsigned{0};
}
// if the floating-point value is a power-of-two frexp normalizes an exact power of two, we can simplify the code
if (__mantissa_ == __power_of_two_mant)
{
const auto __pow2_shift = __shift_ - (__float_digits - 1);
return (__pow2_shift >= __digits) ? _Unsigned{0} : __value >> __pow2_shift;
}
const auto __mantissa = static_cast<_Unsigned>(__mantissa_);
const auto __low = static_cast<_Unsigned>(__value * __mantissa);
const auto __high = ::cuda::mul_hi(__value, __mantissa);
// product = (__high << __digits) | __low
// then product >> shift
if (__shift_ < __digits)
{
return (__high << (__digits - __shift_)) | (__low >> __shift_);
}
return __high >> (__shift_ - __digits);
}
};
template <typename _Tp>
[[nodiscard]] _CCCL_HOST_DEVICE_API bool
__isclose_integer_impl(const _Tp __lhs, const _Tp __rhs, const float __rel_tol, const _Tp __abs_tol) noexcept
{
_CCCL_ASSERT(::cuda::in_range(__rel_tol, 0.0f, 1.0f),
"cuda::isclose: relative tolerance must be in the range [0.0, 1.0]");
_CCCL_ASSERT(::cuda::std::cmp_greater_equal(__abs_tol, _Tp{0}),
"cuda::isclose: absolute tolerance must be non-negative");
using __unsigned_t _CCCL_NODEBUG_ALIAS = ::cuda::std::make_unsigned_t<_Tp>;
const auto __lhs_abs = ::cuda::uabs(__lhs);
const auto __rhs_abs = ::cuda::uabs(__rhs);
const auto __diff = ::cuda::__safe_abs_diff(__lhs, __rhs);
const auto __abs = static_cast<__unsigned_t>(__abs_tol);
const auto __max_abs = ::cuda::std::max(__lhs_abs, __rhs_abs);
const auto __rel_value = ::cuda::__float_ratio{__rel_tol} * __max_abs;
return __diff <= ::cuda::std::max(__abs, __rel_value);
}
//----------------------------------------------------------------------------------------------------------------------
// Public API
// Scalar overloads
//! @brief Checks whether two arithmetic values are close to each other using a relative and absolute tolerance.
//!
//! @param __lhs The first value to compare.
//! @param __rhs The second value to compare.
//! @param __rel_tol The relative tolerance.
//! @param __abs_tol The absolute tolerance.
//! @return True if __lhs and __rhs are close to each other, false otherwise.
_CCCL_TEMPLATE(typename _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp> || ::cuda::is_floating_point_v<_Tp>)
[[nodiscard]] _CCCL_HOST_DEVICE_API bool
isclose(const _Tp __lhs, const _Tp __rhs, const float __rel_tol, const _Tp __abs_tol) noexcept
{
if constexpr (::cuda::std::__cccl_is_integer_v<_Tp>)
{
return ::cuda::__isclose_integer_impl(+__lhs, +__rhs, __rel_tol, +__abs_tol);
}
else
{
using __value_t _CCCL_NODEBUG_ALIAS = __isclose_compare_t<_Tp>;
return ::cuda::__isclose_fp_impl(
static_cast<__value_t>(__lhs), static_cast<__value_t>(__rhs), __rel_tol, static_cast<__value_t>(__abs_tol));
}
}
//! @brief Checks whether two arithmetic values are close to each other using a relative tolerance.
//!
//! @param __lhs The first value to compare.
//! @param __rhs The second value to compare.
//! @param __rel_tol The relative tolerance.
//! @return True if __lhs and __rhs are close to each other, false otherwise.
_CCCL_TEMPLATE(typename _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp> || ::cuda::is_floating_point_v<_Tp>)
[[nodiscard]] _CCCL_HOST_DEVICE_API bool isclose(const _Tp __lhs, const _Tp __rhs, const float __rel_tol) noexcept
{
return ::cuda::isclose(__lhs, __rhs, __rel_tol, _Tp{0});
}
//! @brief Checks whether two arithmetic values are close to each other using the default relative tolerance.
//!
//! @param __lhs The first value to compare.
//! @param __rhs The second value to compare.
//! @return True if __lhs and __rhs are close to each other, false otherwise.
_CCCL_TEMPLATE(typename _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp> || ::cuda::is_floating_point_v<_Tp>)
[[nodiscard]] _CCCL_HOST_DEVICE_API bool isclose(const _Tp __lhs, const _Tp __rhs) noexcept
{
if constexpr (::cuda::std::__cccl_is_integer_v<_Tp>)
{
return __lhs == __rhs;
}
else
{
constexpr auto __rel_tol = ::cuda::__isclose_default_relative_tolerance<_Tp>();
return ::cuda::isclose(__lhs, __rhs, __rel_tol, _Tp{0});
}
}
// Complex overloads
template <typename _Tp, typename _AbsTol, bool = __is_any_complex_v<_Tp>>
inline constexpr bool __isclose_complex_comparison_v = false;
template <typename _Tp, typename _AbsTol>
inline constexpr bool __isclose_complex_comparison_v<_Tp, _AbsTol, true> =
::cuda::std::is_same_v<typename _Tp::value_type, _AbsTol>;
//! @brief Checks whether two complex values are close to each other using a relative and absolute tolerance.
//!
//! @param __lhs The first value to compare.
//! @param __rhs The second value to compare.
//! @param __rel_tol The relative tolerance.
//! @param __abs_tol The absolute tolerance.
//! @return True if __lhs and __rhs are close to each other, false otherwise.
_CCCL_TEMPLATE(typename _ComplexType, typename _AbsTol)
_CCCL_REQUIRES(__isclose_complex_comparison_v<_ComplexType, _AbsTol>)
[[nodiscard]] _CCCL_HOST_DEVICE_API bool
isclose(const _ComplexType& __lhs, const _ComplexType& __rhs, const float __rel_tol, const _AbsTol __abs_tol) noexcept
{
return ::cuda::__isclose_complex_impl(__lhs, __rhs, __rel_tol, __abs_tol);
}
//! @brief Checks whether two complex values are close to each other using a relative tolerance.
//!
//! @param __lhs The first value to compare.
//! @param __rhs The second value to compare.
//! @param __rel_tol The relative tolerance.
//! @return True if __lhs and __rhs are close to each other, false otherwise.
_CCCL_TEMPLATE(typename _ComplexType)
_CCCL_REQUIRES(__is_any_complex_v<_ComplexType>)
[[nodiscard]] _CCCL_HOST_DEVICE_API bool
isclose(const _ComplexType& __lhs, const _ComplexType& __rhs, const float __rel_tol) noexcept
{
using __scalar_t _CCCL_NODEBUG_ALIAS = typename _ComplexType::value_type;
return ::cuda::isclose(__lhs, __rhs, __rel_tol, __scalar_t{0});
}
//! @brief Checks whether two complex values are close to each other using the default relative tolerance.
//!
//! @param __lhs The first value to compare.
//! @param __rhs The second value to compare.
//! @return True if __lhs and __rhs are close to each other, false otherwise.
_CCCL_TEMPLATE(typename _ComplexType)
_CCCL_REQUIRES(__is_any_complex_v<_ComplexType>)
[[nodiscard]] _CCCL_HOST_DEVICE_API bool isclose(const _ComplexType& __lhs, const _ComplexType& __rhs) noexcept
{
using __scalar_t _CCCL_NODEBUG_ALIAS = typename _ComplexType::value_type;
return ::cuda::isclose(__lhs, __rhs, ::cuda::__isclose_default_relative_tolerance<__scalar_t>(), __scalar_t{0});
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___NUMERIC_ISCLOSE_H

248
cccl_upstream/libcudacxx/include/cuda/__numeric/mul_overflow.h Normal file
View File

@@ -0,0 +1,248 @@
//===----------------------------------------------------------------------===//
//
// Part of the libcu++ Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___NUMERIC_MUL_OVERFLOW_H
#define _CUDA___NUMERIC_MUL_OVERFLOW_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__cmath/mul_hi.h>
#include <cuda/__cmath/neg.h>
#include <cuda/__cmath/uabs.h>
#include <cuda/__numeric/overflow_cast.h>
#include <cuda/__numeric/overflow_result.h>
#include <cuda/std/__algorithm/max.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/common_type.h>
#include <cuda/std/__type_traits/conditional.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_same.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__type_traits/is_void.h>
#include <cuda/std/__type_traits/make_nbit_int.h>
#include <cuda/std/__type_traits/num_bits.h>
#include <cuda/std/__utility/cmp.h>
#include <cuda/std/cstdint>
#if _CCCL_COMPILER(MSVC)
# include <intrin.h>
#endif // _CCCL_COMPILER(MSVC)
#include <cuda/std/__cccl/prologue.h>
#if _CCCL_HAS_BUILTIN(__builtin_mul_overflow) || _CCCL_COMPILER(GCC)
# define _CCCL_BUILTIN_MUL_OVERFLOW(...) __builtin_mul_overflow(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__builtin_mul_overflow) || _CCCL_COMPILER(GCC)
// nvc++ < 26.1 doesn't support 128-bit integers and crashes when certain type combinations are used (nvbug 5730860).
#if _CCCL_COMPILER(NVHPC, <, 26, 1)
# undef _CCCL_BUILTIN_MUL_OVERFLOW
#endif // _CCCL_COMPILER(NVHPC, <, 26, 1)
// On ARM64, using the builtin with 128-bit ints result in `undefined reference to __muloti4` with nvc++ and clang < 20.
#if _CCCL_HOST_ARCH(ARM64) && (_CCCL_COMPILER(NVHPC) || _CCCL_COMPILER(CLANG, <, 20))
# undef _CCCL_BUILTIN_MUL_OVERFLOW
#endif // _CCCL_HOST_ARCH(ARM64) && (_CCCL_COMPILER(NVHPC) || _CCCL_COMPILER(CLANG, <, 20))
_CCCL_BEGIN_NAMESPACE_CUDA
template <class _Result, class _Lhs, class _Rhs>
[[nodiscard]] _CCCL_API constexpr overflow_result<_Result> __mul_overflow_generic(_Lhs __lhs, _Rhs __rhs) noexcept
{
using ::cuda::std::__cccl_uintmax_t;
using ::cuda::std::__num_bits_v;
using ::cuda::std::is_signed_v;
// If there is a wider type available, upcast the operands and check for overflow
if constexpr (sizeof(_Lhs) < sizeof(__cccl_uintmax_t) && sizeof(_Rhs) < sizeof(__cccl_uintmax_t))
{
constexpr auto __max_nbits = ::cuda::std::max(__num_bits_v<_Lhs>, __num_bits_v<_Rhs>);
using _Up = ::cuda::std::__make_nbit_int_t<2 * __max_nbits, is_signed_v<_Lhs> || is_signed_v<_Rhs>>;
const auto __result = static_cast<_Up>(__lhs) * static_cast<_Up>(__rhs);
return ::cuda::overflow_cast<_Result>(__result);
}
else if constexpr (is_signed_v<_Lhs> || is_signed_v<_Rhs>)
{
constexpr auto __min = ::cuda::std::numeric_limits<_Result>::min();
constexpr auto __max = ::cuda::std::numeric_limits<_Result>::max();
const auto __negative_result =
(::cuda::std::cmp_greater_equal(__lhs, 0) != ::cuda::std::cmp_greater_equal(__rhs, 0));
const auto __ulhs = __cccl_uintmax_t{::cuda::uabs(__lhs)};
const auto __urhs = __cccl_uintmax_t{::cuda::uabs(__rhs)};
const auto __uresult_lo = __ulhs * __urhs;
const auto __uresult_hi = ::cuda::mul_hi(__ulhs, __urhs);
const auto __uresult_max = __cccl_uintmax_t{::cuda::uabs((__negative_result) ? __min : __max)};
const auto __result = static_cast<_Result>((__negative_result) ? ::cuda::neg(__uresult_lo) : __uresult_lo);
return {__result, __uresult_hi != 0 || __uresult_lo > __uresult_max};
}
else
{
const auto [__result, __overflow] = ::cuda::overflow_cast<_Result>(__lhs * __rhs);
return {__result, __overflow || ::cuda::mul_hi(__cccl_uintmax_t{__lhs}, __cccl_uintmax_t{__rhs}) != 0};
}
}
#if !_CCCL_COMPILER(NVRTC)
template <class _Tp>
[[nodiscard]] _CCCL_HOST_API overflow_result<_Tp> __mul_overflow_host(_Tp __lhs, _Tp __rhs) noexcept
{
// MSVC x86_64 intrinsic branches intentionally collapse to the same generic implementation elsewhere.
// NOLINTBEGIN(bugprone-branch-clone)
if constexpr (::cuda::std::is_signed_v<_Tp>)
{
# if _CCCL_COMPILER(MSVC, >=, 19, 37) && _CCCL_HOST_ARCH(X86_64)
if constexpr (sizeof(_Tp) == sizeof(::cuda::std::int8_t))
{
::cuda::std::int16_t __result;
bool __overflow = ::_mul_full_overflow_i8(__lhs, __rhs, &__result);
return {static_cast<_Tp>(__result), __overflow};
}
else if constexpr (sizeof(_Tp) == sizeof(::cuda::std::int16_t))
{
::cuda::std::int16_t __result;
bool __overflow = ::_mul_overflow_i16(__lhs, __rhs, &__result);
return {__result, __overflow};
}
else if constexpr (sizeof(_Tp) == sizeof(::cuda::std::int32_t))
{
::cuda::std::int32_t __result;
bool __overflow = ::_mul_overflow_i32(__lhs, __rhs, &__result);
return {__result, __overflow};
}
else if constexpr (sizeof(_Tp) == sizeof(::cuda::std::int64_t))
{
::cuda::std::int64_t __result;
bool __overflow = ::_mul_overflow_i64(__lhs, __rhs, &__result);
return {__result, __overflow};
}
else
# endif // _CCCL_COMPILER(MSVC, >=, 19, 37) && _CCCL_HOST_ARCH(X86_64)
{
return ::cuda::__mul_overflow_generic<_Tp>(__lhs, __rhs);
}
}
else // ^^^ signed types ^^^ / vvv unsigned types vvv
{
# if _CCCL_COMPILER(MSVC, >=, 19, 37) && _CCCL_HOST_ARCH(X86_64)
if constexpr (sizeof(_Tp) == sizeof(::cuda::std::uint8_t))
{
::cuda::std::uint16_t __result;
bool __overflow = ::_mul_full_overflow_u8(__lhs, __rhs, &__result);
return {static_cast<_Tp>(__result), __overflow};
}
else if constexpr (sizeof(_Tp) == sizeof(::cuda::std::uint16_t))
{
::cuda::std::uint16_t __lo;
::cuda::std::uint16_t __hi;
bool __overflow = ::_mul_full_overflow_u16(__lhs, __rhs, &__lo, &__hi);
return {__lo, __overflow};
}
else if constexpr (sizeof(_Tp) == sizeof(::cuda::std::uint32_t))
{
::cuda::std::uint32_t __lo;
::cuda::std::uint32_t __hi;
bool __overflow = ::_mul_full_overflow_u32(__lhs, __rhs, &__lo, &__hi);
return {__lo, __overflow};
}
else if constexpr (sizeof(_Tp) == sizeof(::cuda::std::uint64_t))
{
::cuda::std::uint64_t __lo;
::cuda::std::uint64_t __hi;
bool __overflow = ::_mul_full_overflow_u64(__lhs, __rhs, &__lo, &__hi);
return {__lo, __overflow};
}
else
# endif // _CCCL_COMPILER(MSVC, >=, 19, 37) && _CCCL_HOST_ARCH(X86_64)
{
return ::cuda::__mul_overflow_generic<_Tp>(__lhs, __rhs);
}
} // ^^^ unsigned types ^^^
// NOLINTEND(bugprone-branch-clone)
}
#endif // !_CCCL_COMPILER(NVRTC)
_CCCL_TEMPLATE(class _Result = void,
class _Lhs,
class _Rhs,
class _Common = ::cuda::std::common_type_t<_Lhs, _Rhs>,
class _ActResult = ::cuda::std::conditional_t<::cuda::std::is_void_v<_Result>, _Common, _Result>)
_CCCL_REQUIRES((::cuda::std::is_void_v<_Result> || ::cuda::std::__cccl_is_integer_v<_Result>)
_CCCL_AND ::cuda::std::__cccl_is_integer_v<_Lhs> _CCCL_AND ::cuda::std::__cccl_is_integer_v<_Rhs>)
[[nodiscard]] _CCCL_API constexpr overflow_result<_ActResult> mul_overflow(_Lhs __lhs, _Rhs __rhs) noexcept
{
// We want to use __builtin_mul_overflow only in host code. When compiling CUDA source file, we cannot use it in
// constant expressions, because it doesn't work before nvcc 13.1 and is buggy in 13.1. When compiling C++ source
// file, we can use it all the time.
#if defined(_CCCL_BUILTIN_MUL_OVERFLOW)
# if _CCCL_CUDA_COMPILATION()
_CCCL_IF_NOT_CONSTEVAL_DEFAULT
# endif // _CCCL_CUDA_COMPILATION()
{
// nvc++ doesn't fully support 128-bit ints with __builtin_mul_overflow.
# if _CCCL_COMPILER(NVHPC)
if constexpr (sizeof(_ActResult) != 16 && sizeof(_Lhs) != 16 && sizeof(_Rhs) != 16)
# endif // _CCCL_COMPILER(NVHPC)
{
NV_IF_TARGET(NV_IS_HOST, ({
overflow_result<_ActResult> __result{};
__result.overflow = _CCCL_BUILTIN_MUL_OVERFLOW(__lhs, __rhs, &__result.value);
return __result;
}))
}
}
#endif // _CCCL_BUILTIN_MUL_OVERFLOW
// Host fallback + device implementation.
#if _CCCL_CUDA_COMPILATION() || !defined(_CCCL_BUILTIN_MUL_OVERFLOW) || (_CCCL_HAS_INT128() && _CCCL_COMPILER(NVHPC))
using ::cuda::std::is_signed_v;
// If we would check for is_same_v, we would get slow path for e. g. long and long long, even though they represent
// the same range.
constexpr auto __all_same_size = sizeof(_ActResult) == sizeof(_Lhs) && sizeof(_ActResult) == sizeof(_Rhs);
constexpr auto __all_same_sign =
is_signed_v<_ActResult> == is_signed_v<_Lhs> && is_signed_v<_ActResult> == is_signed_v<_Rhs>;
if constexpr (__all_same_size && __all_same_sign)
{
_CCCL_IF_NOT_CONSTEVAL_DEFAULT
{
NV_IF_TARGET(
NV_IS_HOST,
(return ::cuda::__mul_overflow_host(static_cast<_ActResult>(__lhs), static_cast<_ActResult>(__rhs));))
}
}
return ::cuda::__mul_overflow_generic<_ActResult>(__lhs, __rhs);
#endif // needs fallback
}
_CCCL_TEMPLATE(class _Result, class _Lhs, class _Rhs)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Result> _CCCL_AND ::cuda::std::__cccl_is_integer_v<_Lhs>
_CCCL_AND ::cuda::std::__cccl_is_integer_v<_Rhs>)
[[nodiscard]] _CCCL_API constexpr bool mul_overflow(_Result& __result, _Lhs __lhs, _Rhs __rhs) noexcept
{
const auto __overflow_result = ::cuda::mul_overflow<_Result>(__lhs, __rhs);
__result = __overflow_result.value;
return __overflow_result.overflow;
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___NUMERIC_MUL_OVERFLOW_H

96
cccl_upstream/libcudacxx/include/cuda/__numeric/narrow.h Normal file
View File

@@ -0,0 +1,96 @@
//===----------------------------------------------------------------------===//
//
// Part of the libcu++ Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___NUMERIC_NARROW_H
#define _CUDA___NUMERIC_NARROW_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__exception/exception_macros.h>
#include <cuda/std/__host_stdlib/stdexcept>
#include <cuda/std/__type_traits/is_arithmetic.h>
#include <cuda/std/__type_traits/is_constructible.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__utility/forward.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! Uses static_cast to cast a value \p __from to type \p _To. \p _To needs to be constructible from \p _From, and \p
//! implement operator!=. This function is intended to show that narrowing and a potential change of the value is
//! intended. Modelled after `gsl::narrow_cast`. See also the C++ Core Guidelines <a
//! href="https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Res-narrowing">ES.46</a> and <a
//! href="https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Res-casts-named">ES.49</a>.
template <class _To, class _From>
[[nodiscard]] _CCCL_API constexpr _To
narrow_cast(_From&& __from) noexcept(noexcept(static_cast<_To>(::cuda::std::forward<_From>(__from))))
{
return static_cast<_To>(::cuda::std::forward<_From>(__from));
}
#if _CCCL_HAS_EXCEPTIONS()
struct narrowing_error : ::std::runtime_error
{
_CCCL_HOST_API narrowing_error()
: ::std::runtime_error("Narrowing error")
{}
};
#endif // _CCCL_HAS_EXCEPTIONS()
//! Uses static_cast to cast a value \p __from to type \p _To and checks whether the value has changed. \p _To needs
//! to be constructible from \p _From and vice versa, and \p implement operator!=. Throws \ref narrowing_error in host
//! code and traps in device code if the value has changed. Modelled after `gsl::narrow`. See also the C++ Core
//! Guidelines <a href="https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Res-narrowing">ES.46</a> and <a
//! href="https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Res-casts-named">ES.49</a>.
template <class _To, class _From>
[[nodiscard]] _CCCL_API constexpr _To narrow(_From __from)
{
static_assert(::cuda::std::is_constructible_v<_From, _To>);
static_assert(::cuda::std::is_constructible_v<_To, _From>);
const auto __converted = static_cast<_To>(__from);
if (static_cast<_From>(__converted) != __from)
{
_CCCL_THROW(::cuda::narrowing_error);
}
if constexpr (::cuda::std::is_arithmetic_v<_From>)
{
if constexpr (::cuda::std::is_signed_v<_From> && !::cuda::std::is_signed_v<_To>)
{
if (__from < _From{})
{
_CCCL_THROW(::cuda::narrowing_error);
}
}
if constexpr (!::cuda::std::is_signed_v<_From> && ::cuda::std::is_signed_v<_To>)
{
if (__converted < _To{})
{
_CCCL_THROW(::cuda::narrowing_error);
}
}
}
return __converted;
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___NUMERIC_NARROW_H

59
cccl_upstream/libcudacxx/include/cuda/__numeric/overflow_cast.h Normal file
View File

@@ -0,0 +1,59 @@
//===----------------------------------------------------------------------===//
//
// Part of the libcu++ Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___NUMERIC_OVERFLOW_CAST_H
#define _CUDA___NUMERIC_OVERFLOW_CAST_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__numeric/overflow_result.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__utility/cmp.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <typename _From, typename _To>
inline constexpr bool __is_integer_representable_v =
::cuda::std::cmp_less_equal(::cuda::std::numeric_limits<_From>::max(), ::cuda::std::numeric_limits<_To>::max())
&& ::cuda::std::cmp_greater_equal(::cuda::std::numeric_limits<_From>::min(), ::cuda::std::numeric_limits<_To>::min());
//! @brief Casts a number \p __from to a number of type \p _To with overflow detection
//! @param __from The number to cast
//! @return An overflow_result object containing the casted number and a boolean indicating whether an overflow
//! occurred
_CCCL_TEMPLATE(class _To, class _From)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_To> _CCCL_AND ::cuda::std::__cccl_is_cv_integer_v<_From>)
[[nodiscard]] _CCCL_API constexpr overflow_result<_To> overflow_cast(const _From& __from) noexcept
{
bool __overflow = false;
if constexpr (!__is_integer_representable_v<_From, _To>)
{
__overflow = !::cuda::std::in_range<_To>(__from);
}
return overflow_result<_To>{static_cast<_To>(__from), __overflow};
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___NUMERIC_OVERFLOW_CAST_H

43
cccl_upstream/libcudacxx/include/cuda/__numeric/overflow_result.h Normal file
View File

@@ -0,0 +1,43 @@
//===----------------------------------------------------------------------===//
//
// Part of the libcu++ Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___NUMERIC_OVERFLOW_RESULT_H
#define _CUDA___NUMERIC_OVERFLOW_RESULT_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <class _Tp>
struct overflow_result
{
_Tp value;
bool overflow;
_CCCL_API constexpr explicit operator bool() const noexcept
{
return overflow;
}
};
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___NUMERIC_OVERFLOW_RESULT_H

67
cccl_upstream/libcudacxx/include/cuda/__numeric/saturating_add_overflow.h Normal file
View File

@@ -0,0 +1,67 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___NUMERIC_SATURATING_ADD_OVERFLOW_H
#define _CUDA___NUMERIC_SATURATING_ADD_OVERFLOW_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__numeric/add_overflow.h>
#include <cuda/__numeric/overflow_result.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr overflow_result<_Tp> saturating_add_overflow(_Tp __x, _Tp __y) noexcept
{
auto __result = ::cuda::add_overflow(__x, __y);
if (__result.overflow)
{
if constexpr (::cuda::std::is_signed_v<_Tp>)
{
__result.value =
(__y < _Tp{0}) ? ::cuda::std::numeric_limits<_Tp>::min() : ::cuda::std::numeric_limits<_Tp>::max();
}
else
{
__result.value = ::cuda::std::numeric_limits<_Tp>::max();
}
}
return __result;
}
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr bool saturating_add_overflow(_Tp& __result, _Tp __x, _Tp __y) noexcept
{
const auto [__value, __overflow] = ::cuda::saturating_add_overflow(__x, __y);
__result = __value;
return __overflow;
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___NUMERIC_SATURATING_ADD_OVERFLOW_H

61
cccl_upstream/libcudacxx/include/cuda/__numeric/saturating_div_overflow.h Normal file
View File

@@ -0,0 +1,61 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___NUMERIC_SATURATING_DIV_OVERFLOW_H
#define _CUDA___NUMERIC_SATURATING_DIV_OVERFLOW_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__numeric/overflow_result.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr overflow_result<_Tp> saturating_div_overflow(_Tp __x, _Tp __y) noexcept
{
_CCCL_ASSERT(__y != _Tp{0}, "division by zero");
if constexpr (::cuda::std::is_signed_v<_Tp>)
{
if (__x == ::cuda::std::numeric_limits<_Tp>::min() && __y == _Tp{-1})
{
return {::cuda::std::numeric_limits<_Tp>::max(), true};
}
}
return {static_cast<_Tp>(__x / __y), false};
}
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr bool saturating_div_overflow(_Tp& __result, _Tp __x, _Tp __y) noexcept
{
const auto [__value, __overflow] = ::cuda::saturating_div_overflow(__x, __y);
__result = __value;
return __overflow;
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___NUMERIC_SATURATING_DIV_OVERFLOW_H

67
cccl_upstream/libcudacxx/include/cuda/__numeric/saturating_mul_overflow.h Normal file
View File

@@ -0,0 +1,67 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___NUMERIC_SATURATING_MUL_OVERFLOW_H
#define _CUDA___NUMERIC_SATURATING_MUL_OVERFLOW_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__numeric/mul_overflow.h>
#include <cuda/__numeric/overflow_result.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr overflow_result<_Tp> saturating_mul_overflow(_Tp __x, _Tp __y) noexcept
{
auto __result = ::cuda::mul_overflow(__x, __y);
if (__result.overflow)
{
if constexpr (::cuda::std::is_signed_v<_Tp>)
{
__result.value =
((__x < 0) == (__y < 0)) ? ::cuda::std::numeric_limits<_Tp>::max() : ::cuda::std::numeric_limits<_Tp>::min();
}
else
{
__result.value = ::cuda::std::numeric_limits<_Tp>::max();
}
}
return __result;
}
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr bool saturating_mul_overflow(_Tp& __result, _Tp __x, _Tp __y) noexcept
{
const auto [__value, __overflow] = ::cuda::saturating_mul_overflow(__x, __y);
__result = __value;
return __overflow;
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___NUMERIC_SATURATING_MUL_OVERFLOW_H

59
cccl_upstream/libcudacxx/include/cuda/__numeric/saturating_overflow_cast.h Normal file
View File

@@ -0,0 +1,59 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___NUMERIC_SATURATING_OVERFLOW_CAST_H
#define _CUDA___NUMERIC_SATURATING_OVERFLOW_CAST_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__numeric/overflow_result.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__utility/cmp.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
_CCCL_TEMPLATE(class _Up, class _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Up> _CCCL_AND ::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr overflow_result<_Up> saturating_overflow_cast(_Tp __x) noexcept
{
if constexpr (!::cuda::std::in_range<_Up>(::cuda::std::numeric_limits<_Tp>::min()))
{
if (::cuda::std::cmp_less(__x, ::cuda::std::numeric_limits<_Up>::min()))
{
return {::cuda::std::numeric_limits<_Up>::min(), true};
}
}
if constexpr (!::cuda::std::in_range<_Up>(::cuda::std::numeric_limits<_Tp>::max()))
{
if (::cuda::std::cmp_greater(__x, ::cuda::std::numeric_limits<_Up>::max()))
{
return {::cuda::std::numeric_limits<_Up>::max(), true};
}
}
return {static_cast<_Up>(__x), false};
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___NUMERIC_SATURATING_OVERFLOW_CAST_H

67
cccl_upstream/libcudacxx/include/cuda/__numeric/saturating_sub_overflow.h Normal file
View File

@@ -0,0 +1,67 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___NUMERIC_SATURATING_SUB_OVERFLOW_H
#define _CUDA___NUMERIC_SATURATING_SUB_OVERFLOW_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__numeric/overflow_result.h>
#include <cuda/__numeric/sub_overflow.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr overflow_result<_Tp> saturating_sub_overflow(_Tp __x, _Tp __y) noexcept
{
auto __result = ::cuda::sub_overflow(__x, __y);
if (__result.overflow)
{
if constexpr (::cuda::std::is_signed_v<_Tp>)
{
__result.value =
(__y > _Tp{0}) ? ::cuda::std::numeric_limits<_Tp>::min() : ::cuda::std::numeric_limits<_Tp>::max();
}
else
{
__result.value = ::cuda::std::numeric_limits<_Tp>::min();
}
}
return __result;
}
_CCCL_TEMPLATE(class _Tp)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
[[nodiscard]] _CCCL_API constexpr bool saturating_sub_overflow(_Tp& __result, _Tp __x, _Tp __y) noexcept
{
const auto [__value, __overflow] = ::cuda::saturating_sub_overflow(__x, __y);
__result = __value;
return __overflow;
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___NUMERIC_SATURATING_SUB_OVERFLOW_H

422
cccl_upstream/libcudacxx/include/cuda/__numeric/sub_overflow.h Normal file
View File

@@ -0,0 +1,422 @@
//===----------------------------------------------------------------------===//
//
// Part of the libcu++ Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef _CUDA___NUMERIC_SUB_OVERFLOW_H
#define _CUDA___NUMERIC_SUB_OVERFLOW_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/__numeric/overflow_cast.h>
#include <cuda/__numeric/overflow_result.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/common_type.h>
#include <cuda/std/__type_traits/conditional.h>
#include <cuda/std/__type_traits/is_integer.h>
#include <cuda/std/__type_traits/is_signed.h>
#include <cuda/std/__type_traits/is_unsigned.h>
#include <cuda/std/__type_traits/is_void.h>
#include <cuda/std/__type_traits/make_nbit_int.h>
#include <cuda/std/__type_traits/make_signed.h>
#include <cuda/std/__type_traits/make_unsigned.h>
#include <cuda/std/__utility/cmp.h>
#include <nv/target>
#if _CCCL_COMPILER(MSVC) && _CCCL_HOST_ARCH(X86_64)
# include <intrin.h>
#endif // _CCCL_COMPILER(MSVC) && _CCCL_HOST_ARCH(X86_64)
#include <cuda/std/__cccl/prologue.h>
#if _CCCL_CHECK_BUILTIN(builtin_sub_overflow) || _CCCL_COMPILER(GCC)
# define _CCCL_BUILTIN_SUB_OVERFLOW(...) __builtin_sub_overflow(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(builtin_sub_overflow)
// nvc++ < 26.1 doesn't support 128-bit integers and crashes when certain type combinations are used (nvbug 5730860).
#if _CCCL_COMPILER(NVHPC, <, 26, 1)
# undef _CCCL_BUILTIN_SUB_OVERFLOW
#endif // _CCCL_COMPILER(NVHPC, <, 26, 1)
_CCCL_BEGIN_NAMESPACE_CUDA
// Subtracts using unsigned intermediates to avoid signed overflow (UB) before casting to the requested result type.
template <typename _Result, typename _Lhs, typename _Rhs>
[[nodiscard]] _CCCL_API constexpr _Result __sub_as_unsigned(_Lhs __lhs, _Rhs __rhs) noexcept
{
using _UnsignedResult = ::cuda::std::make_unsigned_t<_Result>;
const auto __lhs1 = static_cast<_UnsignedResult>(__lhs);
const auto __rhs1 = static_cast<_UnsignedResult>(__rhs);
return static_cast<_Result>(__lhs1 - __rhs1);
}
// addition with unsigned types to avoid UB, return an unsigned type
template <typename _Result, typename _Lhs, typename _Rhs>
[[nodiscard]] _CCCL_API constexpr _Result __add_as_unsigned(_Lhs __lhs, _Rhs __rhs) noexcept
{
using _UnsignedResult = ::cuda::std::make_unsigned_t<_Result>;
const auto __lhs1 = static_cast<_UnsignedResult>(__lhs);
const auto __rhs1 = static_cast<_UnsignedResult>(__rhs);
return static_cast<_Result>(__lhs1 + __rhs1);
}
template <typename _Tp>
[[nodiscard]] _CCCL_API constexpr overflow_result<_Tp> __sub_overflow_generic_impl(_Tp __lhs, _Tp __rhs) noexcept
{
const auto __sub = ::cuda::__sub_as_unsigned<_Tp>(__lhs, __rhs);
if constexpr (::cuda::std::is_signed_v<_Tp>)
{
return {__sub, (__sub > __lhs) == (__rhs >= _Tp{0})};
}
else
{
return {__sub, __sub > __lhs};
}
}
#if _CCCL_DEVICE_COMPILATION()
template <class _Tp>
[[nodiscard]] _CCCL_DEVICE_API overflow_result<_Tp> __sub_overflow_device(_Tp __lhs, _Tp __rhs) noexcept
{
if constexpr (::cuda::std::is_unsigned_v<_Tp>)
{
using ::cuda::std::uint32_t;
using ::cuda::std::uint64_t;
if constexpr (sizeof(_Tp) < sizeof(uint32_t))
{
const auto __result = uint32_t{__lhs} - uint32_t{__rhs};
return {static_cast<_Tp>(__result), !::cuda::std::in_range<_Tp>(__result)};
}
else if constexpr (sizeof(_Tp) == sizeof(uint32_t))
{
uint32_t __result;
int __overflow;
asm("sub.cc.u32 %0, %2, %3;"
"subc.u32 %1, 0, 0;"
: "=r"(__result), "=r"(__overflow)
: "r"(__lhs), "r"(__rhs));
return {__result, static_cast<bool>(__overflow)};
}
else if constexpr (sizeof(_Tp) == sizeof(uint64_t))
{
uint64_t __result;
int __overflow;
asm("sub.cc.u64 %0, %2, %3;"
"subc.u32 %1, 0, 0;"
: "=l"(__result), "=r"(__overflow)
: "l"(__lhs), "l"(__rhs));
return {__result, static_cast<bool>(__overflow)};
}
# if _CCCL_HAS_INT128()
else if constexpr (sizeof(_Tp) == sizeof(__uint128_t))
{
uint64_t __result_lo;
uint64_t __result_hi;
int __overflow;
asm("sub.cc.u64 %1, %4, %6;"
"subc.cc.u64 %0, %3, %5;"
"subc.u32 %2, 0, 0;"
: "=l"(__result_hi), "=l"(__result_lo), "=r"(__overflow)
: "l"(static_cast<uint64_t>(__lhs >> 64)),
"l"(static_cast<uint64_t>(__lhs)),
"l"(static_cast<uint64_t>(__rhs >> 64)),
"l"(static_cast<uint64_t>(__rhs)));
return {(static_cast<__uint128_t>(__result_hi) << 64) | __result_lo, static_cast<bool>(__overflow)};
}
# endif // _CCCL_HAS_INT128()
else
{
return ::cuda::__sub_overflow_generic_impl(__lhs, __rhs);
}
}
else
{
using ::cuda::std::int32_t;
if constexpr (sizeof(_Tp) < sizeof(int32_t))
{
const auto __result = int32_t{__lhs} - int32_t{__rhs};
return {static_cast<_Tp>(__result), !::cuda::std::in_range<_Tp>(__result)};
}
# if _CCCL_HAS_INT128()
else if constexpr (sizeof(_Tp) == sizeof(__int128_t))
{
using _Up = ::cuda::std::make_unsigned_t<_Tp>;
const auto __uadd_result = ::cuda::__sub_overflow_device(static_cast<_Up>(__lhs), static_cast<_Up>(__rhs));
const auto __result = static_cast<_Tp>(__uadd_result.value);
const auto __overflow = ((__lhs >= 0) != (__rhs >= 0)) && (__uadd_result.overflow != (__result >= 0));
return {__result, __overflow};
}
# endif // _CCCL_HAS_INT128()
else
{
// For 32 and 64 bit types, this seems to be the more efficient path.
return ::cuda::__sub_overflow_generic_impl(__lhs, __rhs);
}
}
}
#endif // _CCCL_DEVICE_COMPILATION()
#if _CCCL_HOST_COMPILATION()
template <class _Tp>
[[nodiscard]] _CCCL_HOST_API overflow_result<_Tp> __sub_overflow_host(_Tp __lhs, _Tp __rhs) noexcept
{
# if _CCCL_COMPILER(MSVC) && _CCCL_HOST_ARCH(X86_64)
if constexpr (sizeof(_Tp) <= 8)
{
# if _CCCL_COMPILER(MSVC, >=, 19, 37)
if constexpr (::cuda::std::is_signed_v<_Tp>)
{
overflow_result<_Tp> __result;
if constexpr (sizeof(_Tp) == 1)
{
__result.overflow = ::_sub_overflow_i8(0, __lhs, __rhs, reinterpret_cast<signed char*>(&__result.value));
}
else if constexpr (sizeof(_Tp) == 2)
{
__result.overflow = ::_sub_overflow_i16(0, __lhs, __rhs, reinterpret_cast<short*>(&__result.value));
}
else if constexpr (sizeof(_Tp) == 4)
{
__result.overflow = ::_sub_overflow_i32(0, __lhs, __rhs, reinterpret_cast<int*>(&__result.value));
}
else if constexpr (sizeof(_Tp) == 8)
{
__result.overflow = ::_sub_overflow_i64(0, __lhs, __rhs, reinterpret_cast<long long*>(&__result.value));
}
return __result;
}
else
# endif // _CCCL_COMPILER(MSVC, >=, 19, 37)
if constexpr (::cuda::std::is_unsigned_v<_Tp>)
{ // unsigned
overflow_result<_Tp> __result;
if constexpr (sizeof(_Tp) == 1)
{
__result.overflow = ::_subborrow_u8(0, __lhs, __rhs, reinterpret_cast<unsigned char*>(&__result.value));
}
else if constexpr (sizeof(_Tp) == 2)
{
__result.overflow = ::_subborrow_u16(0, __lhs, __rhs, reinterpret_cast<unsigned short*>(&__result.value));
}
else if constexpr (sizeof(_Tp) == 4)
{
__result.overflow = ::_subborrow_u32(0, __lhs, __rhs, reinterpret_cast<unsigned int*>(&__result.value));
}
else if constexpr (sizeof(_Tp) == 8)
{
__result.overflow = ::_subborrow_u64(0, __lhs, __rhs, reinterpret_cast<unsigned long long*>(&__result.value));
}
return __result;
}
else
{
return ::cuda::__sub_overflow_generic_impl(__lhs, __rhs);
}
}
else
# endif // ^^^ _CCCL_COMPILER(MSVC) || _CCCL_HOST_ARCH(X86_64) ^^^
{
return ::cuda::__sub_overflow_generic_impl(__lhs, __rhs);
}
}
#endif // _CCCL_HOST_COMPILATION()
template <typename _Tp>
[[nodiscard]] _CCCL_API constexpr overflow_result<_Tp> __sub_overflow_uniform_type(_Tp __lhs, _Tp __rhs) noexcept
{
#if !_CCCL_TILE_COMPILATION() // error: asm statement is unsupported in tile code
_CCCL_IF_NOT_CONSTEVAL_DEFAULT
{
NV_IF_TARGET(NV_IS_DEVICE,
(return ::cuda::__sub_overflow_device(__lhs, __rhs);),
(return ::cuda::__sub_overflow_host(__lhs, __rhs);))
}
#endif // !_CCCL_TILE_COMPILATION()
return ::cuda::__sub_overflow_generic_impl(__lhs, __rhs);
}
template <typename _Result, typename _Lhs, typename _Rhs>
inline constexpr bool __is_sub_representable_v = sizeof(_Result) > sizeof(_Lhs) && sizeof(_Result) > sizeof(_Rhs)
&& ::cuda::std::is_signed_v<_Result>;
/***********************************************************************************************************************
* Public interface
**********************************************************************************************************************/
_CCCL_TEMPLATE(typename _Result = void,
typename _Lhs,
typename _Rhs,
typename _Common = ::cuda::std::common_type_t<_Lhs, _Rhs>,
typename _ActualResult = ::cuda::std::conditional_t<::cuda::std::is_void_v<_Result>, _Common, _Result>)
_CCCL_REQUIRES((::cuda::std::is_void_v<_Result> || ::cuda::std::__cccl_is_integer_v<_Result>)
_CCCL_AND ::cuda::std::__cccl_is_integer_v<_Lhs> _CCCL_AND ::cuda::std::__cccl_is_integer_v<_Rhs>)
[[nodiscard]]
_CCCL_API constexpr overflow_result<_ActualResult> sub_overflow(const _Lhs __lhs, const _Rhs __rhs) noexcept
{
using ::cuda::std::is_same_v;
// We want to use __builtin_sub_overflow only in host code. When compiling CUDA source file, we cannot use it in
// constant expressions, because it doesn't work before nvcc 13.1 and is buggy in 13.1. When compiling C++ source
// file, we can use it all the time.
#if defined(_CCCL_BUILTIN_SUB_OVERFLOW)
# if _CCCL_CUDA_COMPILATION()
_CCCL_IF_NOT_CONSTEVAL_DEFAULT
# endif // _CCCL_CUDA_COMPILATION()
{
// nvc++ doesn't support overflow builtins for 128-bit integers of different signedness.
# if _CCCL_COMPILER(NVHPC)
if constexpr ((sizeof(_ActualResult) != 16 && sizeof(_Lhs) != 16 && sizeof(_Rhs) != 16)
|| (is_same_v<_ActualResult, _Lhs> && is_same_v<_ActualResult, _Rhs>) )
# endif // _CCCL_COMPILER(NVHPC)
{
NV_IF_TARGET(NV_IS_HOST, ({
overflow_result<_ActualResult> __result{};
__result.overflow = _CCCL_BUILTIN_SUB_OVERFLOW(__lhs, __rhs, &__result.value);
return __result;
}))
}
}
#endif // _CCCL_BUILTIN_SUB_OVERFLOW
// Host fallback + device implementation.
#if _CCCL_CUDA_COMPILATION() || !defined(_CCCL_BUILTIN_SUB_OVERFLOW) || (_CCCL_COMPILER(NVHPC) && _CCCL_HAS_INT128())
using ::cuda::std::common_type_t;
using ::cuda::std::is_signed_v;
using ::cuda::std::is_unsigned_v;
using ::cuda::std::make_signed_t;
using ::cuda::std::make_unsigned_t;
using ::cuda::std::numeric_limits;
// shortcut for the case where inputs are representable with the result type
if constexpr (__is_sub_representable_v<_ActualResult, _Lhs, _Rhs>)
{
constexpr auto __max_lhs_rhs = sizeof(_Lhs) > sizeof(_Rhs) ? sizeof(_Lhs) : sizeof(_Rhs);
constexpr auto __max_bits = __max_lhs_rhs * 2 <= sizeof(_ActualResult) ? __max_lhs_rhs * 2 : sizeof(_ActualResult);
using _ComputeType = ::cuda::std::__make_nbit_int_t<__max_bits * 8>;
const auto __lhs1 = static_cast<_ComputeType>(__lhs);
const auto __rhs1 = static_cast<_ComputeType>(__rhs);
const auto __sub = static_cast<_ComputeType>(__lhs1 - __rhs1);
return overflow_result<_ActualResult>{static_cast<_ActualResult>(__sub), false};
}
// all types have the same sign
else if constexpr (is_signed_v<_Lhs> == is_signed_v<_Rhs> && is_signed_v<_Lhs> == is_signed_v<_ActualResult>)
{
using _CommonAll = common_type_t<_Common, _ActualResult>;
const auto __lhs1 = static_cast<_CommonAll>(__lhs);
const auto __rhs1 = static_cast<_CommonAll>(__rhs);
const auto __sub = ::cuda::__sub_overflow_uniform_type(__lhs1, __rhs1);
const auto __ret = ::cuda::overflow_cast<_ActualResult>(__sub.value);
return overflow_result<_ActualResult>{__ret.value, __ret.overflow || __sub.overflow};
}
else if (::cuda::std::cmp_less(__lhs, __rhs)) // lhs < rhs -> negative result
{
if constexpr (is_unsigned_v<_ActualResult>) // if _ActualResult is unsigned, any negative result is an underflow
{
const auto __lhs1 = static_cast<_ActualResult>(__lhs);
const auto __rhs1 = static_cast<_ActualResult>(__rhs);
return overflow_result<_ActualResult>{static_cast<_ActualResult>(__lhs1 - __rhs1), true};
}
else
{
// perform the subtraction as signed (negative result) and check if the result is out of range
// Then, there are two cases depending on the sign of the rhs
using _SignedCommonAll = make_signed_t<common_type_t<_Common, _ActualResult>>;
const auto __sub = ::cuda::__sub_as_unsigned<_SignedCommonAll>(__lhs, __rhs);
constexpr auto __result_min = numeric_limits<_ActualResult>::min();
const auto __is_out_of_range = ::cuda::std::cmp_less(__sub, __result_min);
const auto __sub_ret = static_cast<_ActualResult>(__sub);
const bool __rhs_less_than_zero = !is_unsigned_v<_Rhs> && __rhs < _Rhs{0};
if (__rhs_less_than_zero || __is_out_of_range) // if rhs < 0, lhs - rhs > lhs -> no overflow
{
return overflow_result<_ActualResult>{__sub_ret, __is_out_of_range};
}
else // rhs >= 0 -> lhs - rhs < result_min? -> lhs < result_min + rhs
{
// Now, the problem is to compute 'result_min + rhs' correctly
// note: rhs >= 0, result_min < 0
// * if sizeof(_ActualResult) >= sizeof(_Rhs) or _Rhs is signed, we can use the signed common type
// because the sum is always representable
// * if sizeof(_ActualResult) < sizeof(_Rhs) and _Rhs is unsigned, we can still use the signed common type
// if rhs <= result_max because the sum is always representable, e.g. INT_MIN + uint64_t{UINT_MAX} = INT_MAX
constexpr auto __signed_min = numeric_limits<_ActualResult>::min();
constexpr auto __result_max = numeric_limits<make_signed_t<_ActualResult>>::max();
if (sizeof(_ActualResult) >= sizeof(_Rhs)
|| is_signed_v<_Rhs> || ::cuda::std::cmp_less_equal(__rhs, __result_max))
{
// use __add_as_unsigned to avoid UB with INT_MIN
using _SumType = make_signed_t<common_type_t<_Rhs, _ActualResult>>;
const auto __usum = ::cuda::__add_as_unsigned<_SumType>(__signed_min, __rhs);
const bool __is_underflow = ::cuda::std::cmp_less(__lhs, __usum);
return overflow_result<_ActualResult>{__sub_ret, __is_underflow};
}
else // * otherwise, rhs > result_max and we need to use the unsigned common type
{
using _SumType = make_unsigned_t<common_type_t<_Rhs, _ActualResult>>;
const auto __usum = ::cuda::__add_as_unsigned<_SumType>(__signed_min, __rhs);
const bool __is_underflow = ::cuda::std::cmp_less(__lhs, __usum);
return overflow_result<_ActualResult>{__sub_ret, __is_underflow};
}
}
}
}
else // lhs >= rhs -> positive result
{
// perform the subtraction as unsigned (positive result) and check if the result is out of range
// Then, there are two cases depending on the sign of the rhs
using _UnsignedCommonAll = make_unsigned_t<common_type_t<_Common, _ActualResult>>;
const auto __sub = ::cuda::__sub_as_unsigned<_UnsignedCommonAll>(__lhs, __rhs);
const auto __sub_ret = static_cast<_ActualResult>(__sub);
constexpr auto __result_max = numeric_limits<_ActualResult>::max();
const auto __is_out_of_range = ::cuda::std::cmp_greater(__sub, __result_max);
const bool __is_rhs_ge_zero = is_unsigned_v<_Rhs> || __rhs >= 0;
if (__is_rhs_ge_zero || __is_out_of_range) // rhs >= 0 -> lhs - rhs < lhs -> no overflow
{
return overflow_result<_ActualResult>{__sub_ret, __is_out_of_range};
}
else // lhs >= 0 && rhs < 0 -> lhs - rhs > result_max? -> lhs > result_max + rhs
{
using _Up = make_unsigned_t<common_type_t<_Rhs, _ActualResult>>;
constexpr auto __unsigned_max = numeric_limits<_ActualResult>::max();
const auto __sum = ::cuda::__add_as_unsigned<_Up>(__unsigned_max, __rhs);
const bool __is_overflow = ::cuda::std::cmp_greater(__lhs, __sum);
return overflow_result<_ActualResult>{__sub_ret, __is_overflow};
}
}
#endif // _CCCL_CUDA_COMPILATION() || !_CCCL_BUILTIN_SUB_OVERFLOW || (_CCCL_COMPILER(NVHPC) && _CCCL_HAS_INT128())
}
//! @brief Subtracts two numbers \p __lhs and \p __rhs with overflow detection
_CCCL_TEMPLATE(typename _Result, typename _Lhs, typename _Rhs)
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Result> _CCCL_AND ::cuda::std::__cccl_is_integer_v<_Lhs>
_CCCL_AND ::cuda::std::__cccl_is_integer_v<_Rhs>)
[[nodiscard]] _CCCL_API constexpr bool sub_overflow(_Result& __result, const _Lhs __lhs, const _Rhs __rhs) noexcept
{
const auto __res = ::cuda::sub_overflow<_Result>(__lhs, __rhs);
__result = __res.value;
return __res.overflow;
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif // _CUDA___NUMERIC_SUB_OVERFLOW_H