dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

Browse Source 
CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0
This commit is contained in:
EngineX CI
2026-07-30 09:35:51 +00:00
parent b4d01f481e
commit 56fd68e7dd
8871 changed files with 1454674 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
cccl_upstream/libcudacxx/include/cuda/std/__cccl/architecture.h Normal file
View File

@@ -0,0 +1,128 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_ARCH_H
#define __CCCL_ARCH_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/preprocessor.h>
// The header provides the following macros to determine the host architecture:
//
// _CCCL_HOST_ARCH(ARM64) ARM64
// _CCCL_HOST_ARCH(X86_64) X86 64 bit
// CCCL_HOST_ARCH(ARM64) ARM64
// CCCL_HOST_ARCH(X86_64) X86 64 bit
// Determine the host architecture
// Arm 64-bit
#if (defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) /*emulation*/)
# define _CCCL_HOST_ARCH_ARM64_() 1
#else
# define _CCCL_HOST_ARCH_ARM64_() 0
#endif
// X86 64-bit
// _M_X64 is defined even if we are compiling in Arm64 emulation mode
#if (defined(_M_X64) && !defined(_M_ARM64EC)) || defined(__amd64__) || defined(__x86_64__)
# define _CCCL_HOST_ARCH_X86_64_() 1
#else
# define _CCCL_HOST_ARCH_X86_64_() 0
#endif
#define _CCCL_HOST_ARCH(...) _CCCL_HOST_ARCH_##__VA_ARGS__##_()
//! @def CCCL_HOST_ARCH(ARCH) /* implementation defined */
//!
//! @brief Detect the current host architecture.
//!
//! @param ARCH The name of the host architecture to test.
//!
//! @note This macro is made available when including any libcu++ header. Users that wish to
//! include the smallest possible header for this macro should include `<cuda/std/version>`.
//!
//! For supported host architectures, the macro expands to an implementation-defined true value
//! if the current host architecture matches, or false otherwise. These values may be used in
//! boolean expressions (preprocessor or otherwise), but no other guarantees are made.
//!
//! Available values for `ARCH` include:
//!
//! - ``ARM64``: ARM 64-bit, including MSVC ARM64EC emulation.
//! - ``X86_64``: X86 64-bit. This is false when compiling in MSVC ARM64EC emulation mode.
//!
//! Passing any other value will result in an undefined expansion, which may or may not be
//! diagnosed by the compiler.
//!
//! @par Example
//! @code
//! #define MY_OTHER_MACRO 1
//!
//! // Expansion value can be used in ordinary macro conditionals
//! #if CCCL_HOST_ARCH(X86_64) && MY_OTHER_MACRO
//! // ...
//! #endif
//!
//! // Can be negated as usual
//! #if !CCCL_HOST_ARCH(ARM64)
//! // ...
//! #endif
//! @endcode
//!
//! @return true if the specified host architecture is being compiled for, false otherwise.
#ifdef _CCCL_DOXYGEN_INVOKED
# define CCCL_HOST_ARCH(ARCH) /* implementation defined */
#else
# define CCCL_HOST_ARCH(__arch__) _CCCL_HOST_ARCH_##__arch__##_()
#endif
// Note: the public API is single-arg to constrain the API and allow for future expansion. The
// implementation is duplicated to guard against the architecture targets being accidentally
// defined by the user.
// Determine the endianness
#define _CCCL_ENDIAN_LITTLE() 0xDEAD
#define _CCCL_ENDIAN_BIG() 0xFACE
#define _CCCL_ENDIAN_PDP() 0xBEEF
#if _CCCL_COMPILER(NVRTC) || (_CCCL_COMPILER(MSVC) && (_CCCL_HOST_ARCH(X86_64) || _CCCL_HOST_ARCH(ARM64))) \
|| __LITTLE_ENDIAN__
# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_LITTLE()
#elif __BIG_ENDIAN__
# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_BIG()
#elif defined(__BYTE_ORDER__)
# if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_LITTLE()
# elif __BYTE_ORDER__ == __ORDER_PDP_ENDIAN__
# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_PDP()
# elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_BIG()
# endif // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#elif __has_include(<endian.h>)
# include <endian.h>
# if __BYTE_ORDER == __LITTLE_ENDIAN
# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_LITTLE()
# elif __BYTE_ORDER == __PDP_ENDIAN
# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_PDP()
# elif __BYTE_ORDER == __BIG_ENDIAN
# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_BIG()
# endif // __BYTE_ORDER == __BIG_ENDIAN
#endif // ^^^ has endian.h ^^^
#if !defined(_CCCL_ENDIAN_NATIVE)
_CCCL_WARNING("failed to determine the endianness of the host architecture, defaulting to little-endian")
# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_LITTLE()
#endif // !_CCCL_ENDIAN_NATIVE
#define _CCCL_ENDIAN(_NAME) (_CCCL_ENDIAN_NATIVE() == _CCCL_ENDIAN_##_NAME())
#endif // __CCCL_ARCH_H

169
cccl_upstream/libcudacxx/include/cuda/std/__cccl/assert.h Normal file
View File

@@ -0,0 +1,169 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_ASSERT_H
#define __CCCL_ASSERT_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__cccl/attributes.h>
#include <cuda/std/__cccl/builtin.h>
#include <cuda/std/__cccl/execution_space.h>
#include <cuda/std/__cccl/preprocessor.h>
#if _CCCL_HOSTED()
# include <assert.h>
#endif // _CCCL_HOSTED()
#include <nv/target>
#if defined(_DEBUG) || defined(DEBUG)
# ifndef _CCCL_ENABLE_DEBUG_MODE
# define _CCCL_ENABLE_DEBUG_MODE
# endif // !_CCCL_ENABLE_DEBUG_MODE
#endif // _DEBUG || DEBUG
// Automatically enable assertions when debug mode is enabled
#ifdef _CCCL_ENABLE_DEBUG_MODE
# ifndef CCCL_ENABLE_ASSERTIONS
# define CCCL_ENABLE_ASSERTIONS
# endif // !CCCL_ENABLE_ASSERTIONS
#endif // _CCCL_ENABLE_DEBUG_MODE
//! Ensure that we switch on host assertions when all assertions are enabled
#ifndef CCCL_ENABLE_HOST_ASSERTIONS
# ifdef CCCL_ENABLE_ASSERTIONS
# define CCCL_ENABLE_HOST_ASSERTIONS
# endif // CCCL_ENABLE_ASSERTIONS
#endif // !CCCL_ENABLE_HOST_ASSERTIONS
//! Ensure that we switch on device assertions when all assertions are enabled
#ifndef CCCL_ENABLE_DEVICE_ASSERTIONS
# if defined(CCCL_ENABLE_ASSERTIONS) || defined(__CUDACC_DEBUG__)
# define CCCL_ENABLE_DEVICE_ASSERTIONS
# endif // CCCL_ENABLE_ASSERTIONS
#endif // !CCCL_ENABLE_DEVICE_ASSERTIONS
//! Use the different standard library implementations to implement host side asserts
//! _CCCL_ASSERT_IMPL_HOST should never be used directly
#if _CCCL_OS(QNX)
# define _CCCL_ASSERT_IMPL_HOST(expression, message) ((void) 0)
#elif _CCCL_COMPILER(NVRTC) // There is no host standard library in nvrtc
# define _CCCL_ASSERT_IMPL_HOST(expression, message) ((void) 0)
#elif __has_include(<yvals.h>) && _CCCL_OS(WINDOWS) // Windows uses _STL_VERIFY from <yvals.h>
# include <yvals.h>
# define _CCCL_ASSERT_IMPL_HOST(expression, message) _STL_VERIFY(expression, message)
#else // ^^^ MSVC STL ^^^ / vvv !MSVC STL vvv
# ifdef NDEBUG
// Reintroduce the __assert_fail / __assert_rtn declaration
extern "C" {
# if !_CCCL_CUDA_COMPILER(CLANG)
_CCCL_HOST_DEVICE
# endif // !_CCCL_CUDA_COMPILER(CLANG)
# if _CCCL_OS(APPLE)
void __assert_rtn(const char* __function, const char* __assertion, const char* __file, unsigned int __line) noexcept
__attribute__((__noreturn__));
# else // ^^^ _CCCL_OS(APPLE) ^^^ / vvv !_CCCL_OS(APPLE) ^^^
void __assert_fail(const char* __assertion, const char* __file, unsigned int __line, const char* __function) noexcept
__attribute__((__noreturn__));
# endif // !_CCCL_OS(APPLE)
}
# endif // NDEBUG
# if _CCCL_OS(APPLE)
# define _CCCL_ASSERT_IMPL_HOST(expression, message) \
_CCCL_BUILTIN_EXPECT(static_cast<bool>(expression), 1) \
? (void) 0 : __assert_rtn(__func__, __FILE__, __LINE__, message)
# elif _CCCL_OS(ANDROID)
# define _CCCL_ASSERT_IMPL_HOST(expression, message) \
_CCCL_BUILTIN_EXPECT(static_cast<bool>(expression), 1) \
? (void) 0 : __assert2(__FILE__, __LINE__, __func__, message)
# else // ^^^ _CCCL_OS(APPLE) ^^^ / vvv !_CCCL_OS(APPLE) ^^^
# define _CCCL_ASSERT_IMPL_HOST(expression, message) \
_CCCL_BUILTIN_EXPECT(static_cast<bool>(expression), 1) \
? (void) 0 : __assert_fail(message, __FILE__, __LINE__, __func__)
# endif // !_CCCL_OS(APPLE)
#endif // !MSVC STL
//! Use custom implementations with nvcc on device and the host ones with clang-cuda and nvhpc
//! _CCCL_ASSERT_IMPL_DEVICE should never be used directly
#if _CCCL_OS(QNX) || _CCCL_OS(APPLE)
# define _CCCL_ASSERT_IMPL_DEVICE(expression, message) ((void) 0)
#elif _CCCL_COMPILER(NVRTC)
# define _CCCL_ASSERT_IMPL_DEVICE(expression, message) \
_CCCL_BUILTIN_EXPECT(static_cast<bool>(expression), 1) \
? (void) 0 : __assertfail(message, __FILE__, __LINE__, __func__, sizeof(char))
#elif _CCCL_CUDA_COMPILER(NVCC) //! Use __assert_fail to implement device side asserts
# if _CCCL_COMPILER(MSVC)
# define _CCCL_ASSERT_IMPL_DEVICE(expression, message) \
_CCCL_BUILTIN_EXPECT(static_cast<bool>(expression), 1) \
? (void) 0 : _wassert(_CRT_WIDE(#message), __FILEW__, __LINE__)
# elif _CCCL_OS(ANDROID)
# define _CCCL_ASSERT_IMPL_DEVICE(expression, message) \
_CCCL_BUILTIN_EXPECT(static_cast<bool>(expression), 1) \
? (void) 0 : __assert2(__FILE__, __LINE__, __func__, message)
# else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
# define _CCCL_ASSERT_IMPL_DEVICE(expression, message) \
_CCCL_BUILTIN_EXPECT(static_cast<bool>(expression), 1) \
? (void) 0 : __assert_fail(message, __FILE__, __LINE__, __func__)
# endif // !_CCCL_COMPILER(MSVC)
#elif _CCCL_CUDA_COMPILATION()
# define _CCCL_ASSERT_IMPL_DEVICE(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message)
#else // ^^^ _CCCL_CUDA_COMPILATION() ^^^ / vvv !_CCCL_CUDA_COMPILATION() vvv
# define _CCCL_ASSERT_IMPL_DEVICE(expression, message) ((void) 0)
#endif // !_CCCL_CUDA_COMPILATION()
//! _CCCL_ASSERT_HOST is enabled conditionally depending on CCCL_ENABLE_HOST_ASSERTIONS
#ifdef CCCL_ENABLE_HOST_ASSERTIONS
# define _CCCL_ASSERT_HOST(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message)
#else // ^^^ CCCL_ENABLE_HOST_ASSERTIONS ^^^ / vvv !CCCL_ENABLE_HOST_ASSERTIONS vvv
# define _CCCL_ASSERT_HOST(expression, message) ((void) 0)
#endif // !CCCL_ENABLE_HOST_ASSERTIONS
//! _CCCL_ASSERT_DEVICE is enabled conditionally depending on CCCL_ENABLE_DEVICE_ASSERTIONS
#ifdef CCCL_ENABLE_DEVICE_ASSERTIONS
# define _CCCL_ASSERT_DEVICE(expression, message) _CCCL_ASSERT_IMPL_DEVICE(expression, message)
#else // ^^^ CCCL_ENABLE_DEVICE_ASSERTIONS ^^^ / vvv !CCCL_ENABLE_DEVICE_ASSERTIONS vvv
# define _CCCL_ASSERT_DEVICE(expression, message) ((void) 0)
#endif // !CCCL_ENABLE_DEVICE_ASSERTIONS
//! _CCCL_VERIFY is enabled unconditionally and reserved for critical checks that are required to always be on
//! _CCCL_ASSERT is enabled conditionally depending on CCCL_ENABLE_HOST_ASSERTIONS and CCCL_ENABLE_DEVICE_ASSERTIONS
#if _CCCL_CUDA_COMPILER(NVHPC) // NVHPC can't have different behavior for host and device.
// The host version of the assert will also work in device code.
# define _CCCL_VERIFY(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message)
# if defined(CCCL_ENABLE_HOST_ASSERTIONS) || defined(CCCL_ENABLE_DEVICE_ASSERTIONS)
# define _CCCL_ASSERT(expression, message) _CCCL_ASSERT_HOST(expression, message)
# else
# define _CCCL_ASSERT(expression, message) ((void) 0)
# endif
#elif _CCCL_CUDA_COMPILATION()
# if _CCCL_DEVICE_COMPILATION()
# define _CCCL_VERIFY(expression, message) _CCCL_ASSERT_IMPL_DEVICE(expression, message)
# define _CCCL_ASSERT(expression, message) _CCCL_ASSERT_DEVICE(expression, message)
# else // ^^^ _CCCL_DEVICE_COMPILATION() ^^^ / vvv !_CCCL_DEVICE_COMPILATION() vvv
# define _CCCL_VERIFY(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message)
# define _CCCL_ASSERT(expression, message) _CCCL_ASSERT_HOST(expression, message)
# endif // !_CCCL_DEVICE_COMPILATION()
#else // ^^^ _CCCL_CUDA_COMPILATION() ^^^ / vvv !_CCCL_CUDA_COMPILATION() vvv
# define _CCCL_VERIFY(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message)
# define _CCCL_ASSERT(expression, message) _CCCL_ASSERT_HOST(expression, message)
#endif // !_CCCL_CUDA_COMPILATION()
#endif // __CCCL_ASSERT_H

221
cccl_upstream/libcudacxx/include/cuda/std/__cccl/attributes.h Normal file
View File

@@ -0,0 +1,221 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_ATTRIBUTES_H
#define __CCCL_ATTRIBUTES_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__cccl/diagnostic.h>
#include <cuda/std/__cccl/dialect.h>
#include <cuda/std/__cccl/prologue.h>
#ifdef __has_attribute
# define _CCCL_HAS_ATTRIBUTE(__x) __has_attribute(__x)
#else // ^^^ __has_attribute ^^^ / vvv !__has_attribute vvv
# define _CCCL_HAS_ATTRIBUTE(__x) 0
#endif // !__has_attribute
#ifdef __has_cpp_attribute
# define _CCCL_HAS_CPP_ATTRIBUTE(__x) __has_cpp_attribute(__x)
#else // ^^^ __has_cpp_attribute ^^^ / vvv !__has_cpp_attribute vvv
# define _CCCL_HAS_CPP_ATTRIBUTE(__x) 0
#endif // !__has_cpp_attribute
#ifdef __has_declspec_attribute
# define _CCCL_HAS_DECLSPEC_ATTRIBUTE(__x) __has_declspec_attribute(__x)
#else // ^^^ __has_declspec_attribute ^^^ / vvv !__has_declspec_attribute vvv
# define _CCCL_HAS_DECLSPEC_ATTRIBUTE(__x) 0
#endif // !__has_declspec_attribute
// MSVC needs extra help with empty base classes
#if _CCCL_COMPILER(MSVC) || _CCCL_HAS_DECLSPEC_ATTRIBUTE(empty_bases)
# define _CCCL_DECLSPEC_EMPTY_BASES __declspec(empty_bases)
#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
# define _CCCL_DECLSPEC_EMPTY_BASES
#endif // !_CCCL_COMPILER(MSVC)
#if _CCCL_HAS_ATTRIBUTE(__nodebug__)
# define _CCCL_NODEBUG __attribute__((__nodebug__))
#else // ^^^ _CCCL_HAS_ATTRIBUTE(__nodebug__) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(__nodebug__) vvv
# define _CCCL_NODEBUG
#endif // !_CCCL_HAS_ATTRIBUTE(__nodebug__)
// Debuggers do not step into functions marked with __attribute__((__artificial__)). This
// is useful for small wrapper functions that just dispatch to other functions and that
// are inlined into the caller.
#if _CCCL_HAS_ATTRIBUTE(__artificial__) && !_CCCL_CUDA_COMPILER(NVCC)
# define _CCCL_ARTIFICIAL __attribute__((__artificial__))
#else // ^^^ _CCCL_HAS_ATTRIBUTE(__artificial__) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(__artificial__) vvv
# define _CCCL_ARTIFICIAL
#endif // !_CCCL_HAS_ATTRIBUTE(__artificial__)
// The nodebug attribute flattens aliases down to the actual type rather typename meow<T>::type
#if _CCCL_CUDA_COMPILER(CLANG)
# define _CCCL_NODEBUG_ALIAS _CCCL_NODEBUG
#else // ^^^ _CCCL_CUDA_COMPILER(CLANG) ^^^ / vvv !_CCCL_CUDA_COMPILER(CLANG) vvv
# define _CCCL_NODEBUG_ALIAS
#endif // !_CCCL_CUDA_COMPILER(CLANG)
// _CCCL_ASSUME
// NVCC does not properly respect [[assume()]], so use __builtin_assume, see nvbug5458663
#if _CCCL_CUDA_COMPILER(NVCC) && _CCCL_DEVICE_COMPILATION()
# define _CCCL_ASSUME(...) __builtin_assume(__VA_ARGS__)
#elif _CCCL_HAS_CPP_ATTRIBUTE(assume)
# define _CCCL_ASSUME(...) [[assume(__VA_ARGS__)]]
#else
# define _CCCL_ASSUME(...) _CCCL_BUILTIN_ASSUME(__VA_ARGS__)
#endif
#if _CCCL_TILE_COMPILATION() // nvbug6100910: __builtin_assume is not supported in tile mode
# undef _CCCL_ASSUME
# define _CCCL_ASSUME(...)
#endif // _CCCL_TILE_COMPILATION()
// _CCCL_CONST
#if _CCCL_HAS_CPP_ATTRIBUTE(__gnu__::__const__)
# define _CCCL_CONST [[__gnu__::__const__]]
#else // ^^^ has gnu::const ^^^ / vvv no gnu::const vvv
# define _CCCL_CONST _CCCL_PURE
#endif // ^^^ no gnu::const ^^^
// _CCCL_DIAGNOSE_IF
#if _CCCL_HAS_ATTRIBUTE(__diagnose_if__)
# define _CCCL_DIAGNOSE_IF(_COND, _MSG, _TYPE) __attribute__((__diagnose_if__(_COND, _MSG, _TYPE)))
#else // ^^^ _CCCL_HAS_ATTRIBUTE(diagnose_if) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(diagnose_if) vvv
# define _CCCL_DIAGNOSE_IF(_COND, _MSG, _TYPE)
#endif // !_CCCL_HAS_ATTRIBUTE(diagnose_if)
// _CCCL_INTRINSIC
// MSVC provides a way to mark functions as intrinsic provided the function's body consists of a single
// return statement of a cast expression (e.g., move(x) or forward<T>(u)).
#if _CCCL_COMPILER(MSVC) && _CCCL_HAS_CPP_ATTRIBUTE(msvc::intrinsic)
# define _CCCL_INTRINSIC [[msvc::intrinsic]]
#else
# define _CCCL_INTRINSIC
#endif
// _CCCL_PURE
#if _CCCL_CUDA_COMPILER(NVCC, >=, 12, 5)
# define _CCCL_PURE __nv_pure__
#elif _CCCL_HAS_CPP_ATTRIBUTE(__gnu__::__pure__)
# define _CCCL_PURE [[__gnu__::__pure__]]
#elif _CCCL_COMPILER(MSVC)
# define _CCCL_PURE __declspec(noalias)
#else
# define _CCCL_PURE
#endif
// _CCCL_NO_CFI
#if !_CCCL_COMPILER(GCC)
# define _CCCL_NO_CFI _CCCL_NO_SANITIZE("cfi")
#else
# define _CCCL_NO_CFI
#endif
// _CCCL_NO_SANITIZE
#if _CCCL_HAS_ATTRIBUTE(__no_sanitize__)
# define _CCCL_NO_SANITIZE(_STR) __attribute__((__no_sanitize__(_STR)))
#else // ^^^ _CCCL_HAS_ATTRIBUTE(no_sanitize) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(no_sanitize) vvv
# define _CCCL_NO_SANITIZE(_STR)
#endif // !_CCCL_HAS_ATTRIBUTE(no_sanitize)
// _CCCL_NO_SPECIALIZATIONS
#if _CCCL_HAS_CPP_ATTRIBUTE(clang::__no_specializations__)
# define _CCCL_NO_SPECIALIZATIONS_BECAUSE(_MSG) [[clang::__no_specializations__(_MSG)]]
# define _CCCL_HAS_ATTRIBUTE_NO_SPECIALIZATIONS() 1
#elif _CCCL_HAS_CPP_ATTRIBUTE(msvc::no_specializations)
# define _CCCL_NO_SPECIALIZATIONS_BECAUSE(_MSG) [[msvc::no_specializations(_MSG)]]
# define _CCCL_HAS_ATTRIBUTE_NO_SPECIALIZATIONS() 1
#else // ^^^ has attribute no_specializations ^^^ / vvv hasn't attribute no_specializations vvv
# define _CCCL_NO_SPECIALIZATIONS_BECAUSE(_MSG)
# define _CCCL_HAS_ATTRIBUTE_NO_SPECIALIZATIONS() 0
#endif // ^^^ hasn't attribute no_specializations ^^^
#define _CCCL_NO_SPECIALIZATIONS \
_CCCL_NO_SPECIALIZATIONS_BECAUSE("Users are not allowed to specialize this cccl entity")
// _CCCL_LIFETIMEBOUND
#if _CCCL_HAS_CPP_ATTRIBUTE(clang::lifetimebound) || _CCCL_COMPILER(CLANG)
# define _CCCL_LIFETIMEBOUND [[clang::lifetimebound]]
#elif _CCCL_HAS_CPP_ATTRIBUTE(msvc::lifetimebound) || _CCCL_COMPILER(MSVC, >=, 19, 37)
# define _CCCL_LIFETIMEBOUND [[msvc::lifetimebound]]
#else
# define _CCCL_LIFETIMEBOUND
#endif
// _CCCL_NO_UNIQUE_ADDRESS
#if _CCCL_COMPILER(MSVC) || _CCCL_HAS_CPP_ATTRIBUTE(no_unique_address) < 201803L
// MSVC implementation has lead to multiple issues with silent runtime corruption when passing data into kernels
# define _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() 0
# define _CCCL_NO_UNIQUE_ADDRESS
#elif _CCCL_HAS_CPP_ATTRIBUTE(no_unique_address)
# define _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() 1
# define _CCCL_NO_UNIQUE_ADDRESS [[no_unique_address]]
#else
# define _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() 0
# define _CCCL_NO_UNIQUE_ADDRESS
#endif
// Passing objects with nested [[no_unique_address]] to kernels leads to data corruption.
// This is caused by cudafe++ not honoring [[no_unique_address]] when compiling for C++17
// with clang as the host compiler. See nvbug 5265027 for more details.
#if _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() && _CCCL_COMPILER(CLANG) && _CCCL_STD_VER < 2020 \
&& _CCCL_CUDA_COMPILER(NVCC)
# undef _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS
# undef _CCCL_NO_UNIQUE_ADDRESS
# define _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() 0
# define _CCCL_NO_UNIQUE_ADDRESS
#endif // _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() && _CCCL_COMPILER(CLANG)
// _CCCL_PREFERRED_NAME
#if _CCCL_HAS_ATTRIBUTE(__preferred_name__)
# define _CCCL_PREFERRED_NAME(x) __attribute__((__preferred_name__(x)))
#else
# define _CCCL_PREFERRED_NAME(x)
#endif
#if _CCCL_HAS_ATTRIBUTE(__require_constant_initialization__)
# define _CCCL_REQUIRE_CONSTANT_INITIALIZATION __attribute__((__require_constant_initialization__))
#else
# define _CCCL_REQUIRE_CONSTANT_INITIALIZATION
#endif
// _CCCL_RESTRICT
#if _CCCL_COMPILER(MSVC) // vvv _CCCL_COMPILER(MSVC) vvv
# define _CCCL_RESTRICT __restrict
#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
# define _CCCL_RESTRICT __restrict__
#endif // ^^^ !_CCCL_COMPILER(MSVC) ^^^
#include <cuda/std/__cccl/epilogue.h>
#endif // __CCCL_ATTRIBUTES_H

474
cccl_upstream/libcudacxx/include/cuda/std/__cccl/builtin.h Normal file
View File

@@ -0,0 +1,474 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_BUILTIN_H
#define __CCCL_BUILTIN_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/preprocessor.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__cccl/cuda_capabilities.h>
#include <cuda/std/__cccl/extended_data_types.h>
#include <cuda/std/__cccl/host_std_lib.h>
//! This file consolidates all compiler builtin detection for CCCL.
//!
//! To work around older compilers not supporting `__has_builtin` we use `_CCCL_CHECK_BUILTIN` that detects more
//! cases
//!
//! * We work around old clang versions (before clang-10) not supporting __has_builtin via _CCCL_CHECK_BUILTIN
//! * We work around old intel versions (before 2021.3) not supporting __has_builtin via _CCCL_CHECK_BUILTIN
//! * We work around old nvhpc versions (before 2022.11) not supporting __has_builtin via _CCCL_CHECK_BUILTIN
//! * MSVC needs manual handling, has no real way of checking builtins so all is manual
//! * GCC needs manual handling, before gcc-10 as that finally supports __has_builtin
//!
//! In case compiler support for a builtin is advertised but leads to regressions we explicitly undef the macro
//!
//! Finally, because `_CCCL_CHECK_BUILTIN` may lead to false positives, we move detection of new builtins over towards
//! just using _CCCL_HAS_BUILTIN
#ifdef __has_builtin
# define _CCCL_HAS_BUILTIN(__x) __has_builtin(__x)
#else // ^^^ __has_builtin ^^^ / vvv !__has_builtin vvv
# define _CCCL_HAS_BUILTIN(__x) 0
#endif // !__has_builtin
#ifdef __has_feature
# define _CCCL_HAS_FEATURE(__x) __has_feature(__x)
#else // ^^^ __has_feature ^^^ / vvv !__has_feature vvv
# define _CCCL_HAS_FEATURE(__x) 0
#endif // !__has_feature
// '__is_identifier' returns '0' if '__x' is a reserved identifier provided by the compiler and '1' otherwise.
#ifdef __is_identifier
# define _CCCL_IS_IDENTIFIER(__x) __is_identifier(__x)
#else // ^^^ __is_identifier ^^^ / vvv !__is_identifier vvv
# define _CCCL_IS_IDENTIFIER(__x) 1
#endif // !__is_identifier
#define _CCCL_HAS_KEYWORD(__x) !(_CCCL_IS_IDENTIFIER(__x))
// https://bugs.llvm.org/show_bug.cgi?id=44517
#define _CCCL_CHECK_BUILTIN(__x) (_CCCL_HAS_BUILTIN(__##__x) || _CCCL_HAS_KEYWORD(__##__x) || _CCCL_HAS_FEATURE(__x))
// NVCC has issues with function pointers
#if _CCCL_HAS_BUILTIN(__add_lvalue_reference) && _CCCL_CUDA_COMPILER(CLANG)
# define _CCCL_BUILTIN_ADD_LVALUE_REFERENCE(...) __add_lvalue_reference(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__add_lvalue_reference)
// NVCC has issues with function pointers
#if _CCCL_HAS_BUILTIN(__add_pointer) && _CCCL_CUDA_COMPILER(CLANG)
# define _CCCL_BUILTIN_ADD_POINTER(...) __add_pointer(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__add_pointer)
// NVCC has issues with function pointers
#if _CCCL_HAS_BUILTIN(__add_rvalue_reference) && _CCCL_CUDA_COMPILER(CLANG)
# define _CCCL_BUILTIN_ADD_RVALUE_REFERENCE(...) __add_rvalue_reference(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__add_rvalue_reference)
// TODO: Enable using the builtin __array_rank when https://llvm.org/PR57133 is resolved
#if 0 // _CCCL_CHECK_BUILTIN(array_rank)
# define _CCCL_BUILTIN_ARRAY_RANK(...) __array_rank(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(array_rank)
// nvhpc has a bug where it supports __builtin_addressof but does not mark it via _CCCL_CHECK_BUILTIN
#if _CCCL_CHECK_BUILTIN(builtin_addressof) || _CCCL_COMPILER(GCC, >=, 7) || _CCCL_COMPILER(MSVC) \
|| _CCCL_COMPILER(NVHPC) || _CCCL_COMPILER(NVRTC, >=, 12, 3)
# define _CCCL_BUILTIN_ADDRESSOF(...) __builtin_addressof(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(builtin_addressof)
#if _CCCL_CHECK_BUILTIN(builtin_assume) || _CCCL_COMPILER(CLANG) || _CCCL_COMPILER(NVHPC)
# define _CCCL_BUILTIN_ASSUME(...) __builtin_assume(__VA_ARGS__)
#elif _CCCL_COMPILER(GCC, >=, 13)
# define _CCCL_BUILTIN_ASSUME(...) __attribute__((__assume__(__VA_ARGS__)))
#elif _CCCL_COMPILER(MSVC)
# define _CCCL_BUILTIN_ASSUME(...) __assume(__VA_ARGS__)
#else
# define _CCCL_BUILTIN_ASSUME(...)
#endif // _CCCL_CHECK_BUILTIN(builtin_assume)
#if _CCCL_TILE_COMPILATION() // nvbug6100910: __builtin_assume is not supported in tile mode
# undef _CCCL_BUILTIN_ASSUME
# define _CCCL_BUILTIN_ASSUME(...)
#endif // _CCCL_TILE_COMPILATION()
#if _CCCL_HAS_BUILTIN(__builtin_assume_aligned) || _CCCL_COMPILER(MSVC, >=, 19, 23) || _CCCL_COMPILER(GCC)
# define _CCCL_BUILTIN_ASSUME_ALIGNED(...) __builtin_assume_aligned(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__builtin_assume_aligned)
#if _CCCL_CHECK_BUILTIN(builtin_constant_p) || _CCCL_COMPILER(GCC)
# define _CCCL_BUILTIN_CONSTANT_P(...) __builtin_constant_p(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(builtin_constant_p)
#if _CCCL_CHECK_BUILTIN(builtin_expect) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC)
# define _CCCL_BUILTIN_EXPECT(_EXPR, _VAL) __builtin_expect(_EXPR, _VAL)
#else // ^^^ has __builtin_expect ^^^ / vvv no __builtin_expect vvv
# define _CCCL_BUILTIN_EXPECT(_EXPR, _VAL) (_EXPR)
#endif // ^^^ no __builtin_expect ^^^
#if _CCCL_TILE_COMPILATION() // nvbug6100927: __builtin_expect is unsupported in tile mode
# undef _CCCL_BUILTIN_EXPECT
# define _CCCL_BUILTIN_EXPECT(_EXPR, _VAL) (_EXPR)
#endif // _CCCL_TILE_COMPILATION()
#if _CCCL_CHECK_BUILTIN(builtin_huge_valf) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10)
# define _CCCL_BUILTIN_HUGE_VALF() __builtin_huge_valf()
#endif // _CCCL_CHECK_BUILTIN(builtin_huge_valf)
#if _CCCL_CHECK_BUILTIN(builtin_huge_val) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10)
# define _CCCL_BUILTIN_HUGE_VAL() __builtin_huge_val()
#endif // _CCCL_CHECK_BUILTIN(builtin_huge_val)
#if _CCCL_CHECK_BUILTIN(builtin_huge_vall) || _CCCL_COMPILER(GCC, <, 10)
# define _CCCL_BUILTIN_HUGE_VALL() __builtin_huge_vall()
#elif _CCCL_COMPILER(MSVC)
# define _CCCL_BUILTIN_HUGE_VALL() static_cast<long double>(__builtin_huge_val())
#endif // _CCCL_CHECK_BUILTIN(builtin_huge_vall)
#if _CCCL_HAS_FLOAT128()
# if _CCCL_CHECK_BUILTIN(builtin_huge_valf128) || _CCCL_COMPILER(GCC, >=, 7)
# define _CCCL_BUILTIN_HUGE_VALF128() __builtin_huge_valf128()
# endif // _CCCL_CHECK_BUILTIN(builtin_huge_valf128) || _CCCL_COMPILER(GCC, >=, 7)
// nvcc does not implement __builtin_huge_valf128
# if _CCCL_CUDA_COMPILER(NVCC)
# undef _CCCL_BUILTIN_HUGE_VALF128
# endif // _CCCL_CUDA_COMPILER(NVCC)
#endif // _CCCL_HAS_FLOAT128()
#if _CCCL_CHECK_BUILTIN(builtin_is_constant_evaluated) || _CCCL_COMPILER(GCC, >=, 9) || _CCCL_COMPILER(MSVC, >, 19, 24)
# define _CCCL_BUILTIN_IS_CONSTANT_EVALUATED(...) __builtin_is_constant_evaluated(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(builtin_is_constant_evaluated)
#if _CCCL_TILE_COMPILATION() // nvbug6067464: __builtin_is_constant_evaluated is unsupported in tile mode
# undef _CCCL_BUILTIN_IS_CONSTANT_EVALUATED
#endif // _CCCL_TILE_COMPILATION()
#if _CCCL_CHECK_BUILTIN(builtin_is_corresponding_member)
# define _CCCL_BUILTIN_IS_CORRESPONDING_MEMBER(_C1, _C2, _MPtr1, _MPtr2) \
__builtin_is_corresponding_member(_MPtr1, _MPtr2)
#elif _CCCL_COMPILER(MSVC, >=, 19, 29)
// using __is_corresponding_member with msvc outside of constexpr context causes linker errors, see
// https://developercommunity.visualstudio.com/t/Using-compiler-builtins-causes-linking-n/10888080
// # define _CCCL_BUILTIN_IS_CORRESPONDING_MEMBER(_C1, _C2, _MPtr1, _MPtr2) __is_corresponding_member(_C1, _C2, _MPtr1,
// _MPtr2)
#endif // ^^^ _CCCL_COMPILER(MSVC, >=, 19, 29) ^^^
#if _CCCL_CHECK_BUILTIN(builtin_is_pointer_interconvertible_with_class)
# define _CCCL_BUILTIN_IS_POINTER_INTERCONVERTIBLE_WITH_CLASS(_S, _MPtr) \
__builtin_is_pointer_interconvertible_with_class(_MPtr)
#elif _CCCL_COMPILER(MSVC, >=, 19, 29)
// using __is_pointer_interconvertible_with_class with msvc outside of constexpr context causes linker errors, see
// https://developercommunity.visualstudio.com/t/Using-compiler-builtins-causes-linking-n/10888080
// # define _CCCL_BUILTIN_IS_POINTER_INTERCONVERTIBLE_WITH_CLASS(_S, _MPtr)
// __is_pointer_interconvertible_with_class(_S, _MPtr)
#endif // ^^^ _CCCL_COMPILER(MSVC, >=, 19, 29) ^^^
#if _CCCL_CHECK_BUILTIN(builtin_is_virtual_base_of)
# define _CCCL_BUILTIN_IS_VIRTUAL_BASE_OF(...) __builtin_is_virtual_base_of(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(builtin_is_virtual_base_of)
// nvcc < 13.3 doesn't implement __builtin_is_virtual_base_of
#if _CCCL_CUDA_COMPILER(NVCC, <, 13, 3)
# undef _CCCL_BUILTIN_IS_VIRTUAL_BASE_OF
#endif // _CCCL_CUDA_COMPILER(NVCC, <, 13, 3)
#if _CCCL_CHECK_BUILTIN(builtin_nanf) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10)
# define _CCCL_BUILTIN_NANF(...) __builtin_nanf(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(builtin_nanf)
#if _CCCL_CHECK_BUILTIN(builtin_nan) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10)
# define _CCCL_BUILTIN_NAN(...) __builtin_nan(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(builtin_nan)
#if _CCCL_CHECK_BUILTIN(builtin_nanl) || _CCCL_COMPILER(GCC, <, 10)
# define _CCCL_BUILTIN_NANL(...) __builtin_nanl(__VA_ARGS__)
#elif _CCCL_COMPILER(MSVC)
# define _CCCL_BUILTIN_NANL(...) static_cast<long double>(__builtin_nan(__VA_ARGS__))
#endif // _CCCL_CHECK_BUILTIN(builtin_nanl)
#if _CCCL_HAS_FLOAT128()
# if _CCCL_CHECK_BUILTIN(builtin_nanf128) || _CCCL_COMPILER(GCC, >=, 7)
# define _CCCL_BUILTIN_NANF128(...) __builtin_nanf128(__VA_ARGS__)
# endif // _CCCL_CHECK_BUILTIN(builtin_nanf128) || _CCCL_COMPILER(GCC, >=, 7)
// nvcc does not implement __builtin_nanf128
# if _CCCL_CUDA_COMPILER(NVCC)
# undef _CCCL_BUILTIN_NANF128
# endif // _CCCL_CUDA_COMPILER(NVCC)
#endif // _CCCL_HAS_FLOAT128()
#if _CCCL_CHECK_BUILTIN(builtin_nansf) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10)
# define _CCCL_BUILTIN_NANSF(...) __builtin_nansf(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(builtin_nansf)
#if _CCCL_CHECK_BUILTIN(builtin_nans) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10)
# define _CCCL_BUILTIN_NANS(...) __builtin_nans(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(builtin_nans)
#if _CCCL_CHECK_BUILTIN(builtin_nansl) || _CCCL_COMPILER(GCC, <, 10)
# define _CCCL_BUILTIN_NANSL(...) __builtin_nansl(__VA_ARGS__)
#elif _CCCL_COMPILER(MSVC)
# define _CCCL_BUILTIN_NANSL(...) static_cast<long double>(__builtin_nans(__VA_ARGS__))
#endif // _CCCL_CHECK_BUILTIN(builtin_nansl)
#if _CCCL_HAS_FLOAT128()
# if _CCCL_CHECK_BUILTIN(builtin_nansf128) || _CCCL_COMPILER(GCC, >=, 7)
# define _CCCL_BUILTIN_NANSF128(...) __builtin_nansf128(__VA_ARGS__)
# endif // _CCCL_CHECK_BUILTIN(builtin_nansf128) || _CCCL_COMPILER(GCC, >=, 7)
// nvcc does not implement __builtin_nansf128
# if _CCCL_CUDA_COMPILER(NVCC)
# undef _CCCL_BUILTIN_NANSF128
# endif // _CCCL_CUDA_COMPILER(NVCC)
#endif // _CCCL_HAS_FLOAT128()
#if _CCCL_CHECK_BUILTIN(builtin_memcmp) || _CCCL_COMPILER(GCC) || _CCCL_COMPILER(MSVC, >=, 19, 28)
# define _CCCL_BUILTIN_MEMCMP(...) __builtin_memcmp(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(builtin_memcmp) || _CCCL_COMPILER(GCC) || _CCCL_COMPILER(MSVC, >=, 19, 28)
#if _CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(CLANG)
# undef _CCCL_BUILTIN_MEMCMP
#endif // _CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(CLANG)
#if _CCCL_CHECK_BUILTIN(builtin_memmove) || _CCCL_COMPILER(GCC)
# define _CCCL_BUILTIN_MEMMOVE(...) __builtin_memmove(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(builtin_memmove) || _CCCL_COMPILER(GCC)
#if _CCCL_CUDA_COMPILER(NVCC)
# undef _CCCL_BUILTIN_MEMMOVE
#endif // _CCCL_CUDA_COMPILER(NVCC)
#if _CCCL_CHECK_BUILTIN(builtin_operator_new) && _CCCL_CHECK_BUILTIN(builtin_operator_delete) \
&& _CCCL_CUDA_COMPILER(CLANG)
# define _CCCL_BUILTIN_OPERATOR_DELETE(...) __builtin_operator_delete(__VA_ARGS__)
# define _CCCL_BUILTIN_OPERATOR_NEW(...) __builtin_operator_new(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(builtin_operator_new) && _CCCL_CHECK_BUILTIN(builtin_operator_delete)
#if _CCCL_CHECK_BUILTIN(builtin_prefetch) || _CCCL_COMPILER(GCC)
# define _CCCL_BUILTIN_PREFETCH(...) NV_IF_TARGET(NV_IS_HOST, __builtin_prefetch(__VA_ARGS__);)
#else
# define _CCCL_BUILTIN_PREFETCH(...)
#endif // _CCCL_CHECK_BUILTIN(builtin_prefetch)
#if _CCCL_HAS_BUILTIN(__decay) && _CCCL_CUDA_COMPILER(CLANG)
# define _CCCL_BUILTIN_DECAY(...) __decay(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__decay) && clang-cuda
#if _CCCL_CHECK_BUILTIN(has_nothrow_assign) || _CCCL_COMPILER(GCC, >=, 4, 3) || _CCCL_COMPILER(MSVC) \
|| _CCCL_COMPILER(NVRTC)
# define _CCCL_BUILTIN_HAS_NOTHROW_ASSIGN(...) __has_nothrow_assign(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(has_nothrow_assign) && gcc >= 4.3
#if _CCCL_CHECK_BUILTIN(has_nothrow_constructor) || _CCCL_COMPILER(GCC, >=, 4, 3) || _CCCL_COMPILER(MSVC) \
|| _CCCL_COMPILER(NVRTC)
# define _CCCL_BUILTIN_HAS_NOTHROW_CONSTRUCTOR(...) __has_nothrow_constructor(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(has_nothrow_constructor) && gcc >= 4.3
#if _CCCL_CHECK_BUILTIN(has_nothrow_copy) || _CCCL_COMPILER(GCC, >=, 4, 3) || _CCCL_COMPILER(MSVC) \
|| _CCCL_COMPILER(NVRTC)
# define _CCCL_BUILTIN_HAS_NOTHROW_COPY(...) __has_nothrow_copy(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(has_nothrow_copy) && gcc >= 4.3
#if _CCCL_HAS_BUILTIN(__integer_pack)
# define _CCCL_BUILTIN_INTEGER_PACK(...) __integer_pack(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__integer_pack)
#if _CCCL_CHECK_BUILTIN(is_array)
# define _CCCL_BUILTIN_IS_ARRAY(...) __is_array(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(is_array)
// clang prior to clang-19 gives wrong results for __is_array of _Tp[0]
#if _CCCL_COMPILER(CLANG, <, 19)
# undef _CCCL_BUILTIN_IS_ARRAY
#endif // clang < 19
#if _CCCL_CHECK_BUILTIN(is_assignable) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, >=, 9)
# define _CCCL_BUILTIN_IS_ASSIGNABLE(...) __is_assignable(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(is_assignable) && gcc >= 9.0
#if _CCCL_CHECK_BUILTIN(is_constructible) || _CCCL_COMPILER(GCC, >=, 8) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC)
# define _CCCL_BUILTIN_IS_CONSTRUCTIBLE(...) __is_constructible(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(is_constructible) && gcc >= 8.0
#if _CCCL_CHECK_BUILTIN(is_convertible_to) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC)
# define _CCCL_BUILTIN_IS_CONVERTIBLE_TO(...) __is_convertible_to(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(is_convertible_to)
#if _CCCL_CHECK_BUILTIN(is_destructible) || _CCCL_COMPILER(MSVC)
# define _CCCL_BUILTIN_IS_DESTRUCTIBLE(...) __is_destructible(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(is_destructible)
#if _CCCL_CHECK_BUILTIN(is_layout_compatible) || _CCCL_COMPILER(MSVC, >=, 19, 29)
# define _CCCL_BUILTIN_IS_LAYOUT_COMPATIBLE(...) __is_layout_compatible(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(is_layout_compatible) || _CCCL_COMPILER(MSVC, >=, 19, 29)
#if _CCCL_CHECK_BUILTIN(is_lvalue_reference)
# define _CCCL_BUILTIN_IS_LVALUE_REFERENCE(...) __is_lvalue_reference(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(is_lvalue_reference)
#if _CCCL_HAS_BUILTIN(__is_member_function_pointer)
# define _CCCL_BUILTIN_IS_MEMBER_FUNCTION_POINTER(...) __is_member_function_pointer(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__is_member_function_pointer)
#if _CCCL_HAS_BUILTIN(__is_member_object_pointer)
# define _CCCL_BUILTIN_IS_MEMBER_OBJECT_POINTER(...) __is_member_object_pointer(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__is_member_object_pointer)
#if _CCCL_HAS_BUILTIN(__is_member_pointer)
# define _CCCL_BUILTIN_IS_MEMBER_POINTER(...) __is_member_pointer(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__is_member_pointer)
#if _CCCL_CHECK_BUILTIN(is_nothrow_assignable) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC)
# define _CCCL_BUILTIN_IS_NOTHROW_ASSIGNABLE(...) __is_nothrow_assignable(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(is_nothrow_assignable)
#if _CCCL_CHECK_BUILTIN(is_nothrow_constructible) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC)
# define _CCCL_BUILTIN_IS_NOTHROW_CONSTRUCTIBLE(...) __is_nothrow_constructible(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(is_nothrow_constructible)
#if _CCCL_CHECK_BUILTIN(is_nothrow_destructible) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC)
# define _CCCL_BUILTIN_IS_NOTHROW_DESTRUCTIBLE(...) __is_nothrow_destructible(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(is_nothrow_destructible)
#if _CCCL_CHECK_BUILTIN(is_object)
# define _CCCL_BUILTIN_IS_OBJECT(...) __is_object(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(is_object)
// Disabled due to libstdc++ conflict
#if 0 // _CCCL_HAS_BUILTIN(__is_pointer)
# define _CCCL_BUILTIN_IS_POINTER(...) __is_pointer(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__is_pointer)
#if _CCCL_CHECK_BUILTIN(is_pointer_interconvertible_base_of) || _CCCL_COMPILER(MSVC, >=, 19, 29)
# define _CCCL_BUILTIN_IS_POINTER_INTERCONVERTIBLE_BASE_OF(...) __is_pointer_interconvertible_base_of(__VA_ARGS__)
#endif // _CCCL_CHECK_BUILTIN(is_pointer_interconvertible_base_of) || _CCCL_COMPILER(MSVC, >=, 19, 29)
#if _CCCL_HAS_BUILTIN(__is_reference)
# define _CCCL_BUILTIN_IS_REFERENCE(...) __is_reference(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__is_reference)
// Disabled due to libstdc++ conflict
#if 0 // _CCCL_HAS_BUILTIN(__is_referenceable)
# define _CCCL_BUILTIN_IS_REFERENCEABLE(...) __is_referenceable(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__is_referenceable)
#if _CCCL_HAS_BUILTIN(__is_rvalue_reference)
# define _CCCL_BUILTIN_IS_RVALUE_REFERENCE(...) __is_rvalue_reference(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__is_rvalue_reference)
// Disabled due to libstdc++ conflict
#if 0 // _CCCL_HAS_BUILTIN(__is_scalar)
# define _CCCL_BUILTIN_IS_SCALAR(...) __is_scalar(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__is_scalar)
#if _CCCL_CHECK_BUILTIN(make_integer_seq) || _CCCL_COMPILER(MSVC, >=, 19, 23)
# define _CCCL_BUILTIN_MAKE_INTEGER_SEQ(...) __make_integer_seq<__VA_ARGS__>
#endif // _CCCL_CHECK_BUILTIN(make_integer_seq)
#if _CCCL_HAS_BUILTIN(__reference_constructs_from_temporary)
# define _CCCL_BUILTIN_REFERENCE_CONSTRUCTS_FROM_TEMPORARY(...) __reference_constructs_from_temporary(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__reference_constructs_from_temporary)
#if _CCCL_HAS_BUILTIN(__reference_converts_from_temporary)
# define _CCCL_BUILTIN_REFERENCE_CONVERTS_FROM_TEMPORARY(...) __reference_converts_from_temporary(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__reference_converts_from_temporary)
#if _CCCL_HAS_BUILTIN(__remove_const) && _CCCL_CUDA_COMPILER(CLANG)
# define _CCCL_BUILTIN_REMOVE_CONST(...) __remove_const(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__remove_const)
#if _CCCL_HAS_BUILTIN(__remove_cv) && _CCCL_CUDA_COMPILER(CLANG)
# define _CCCL_BUILTIN_REMOVE_CV(...) __remove_cv(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__remove_cv)
#if _CCCL_HAS_BUILTIN(__remove_cvref) && _CCCL_CUDA_COMPILER(CLANG)
# define _CCCL_BUILTIN_REMOVE_CVREF(...) __remove_cvref(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__remove_cvref)
#if _CCCL_COMPILER(NVRTC, <, 12, 4) // NVRTC below 12.4 fails to properly compile that builtin
# undef _CCCL_BUILTIN_REMOVE_CVREF
#endif // _CCCL_COMPILER(NVRTC, <, 12, 4)
#if _CCCL_HAS_BUILTIN(__remove_extent) && _CCCL_CUDA_COMPILER(CLANG)
# define _CCCL_BUILTIN_REMOVE_EXTENT(...) __remove_extent(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__remove_extent)
#if _CCCL_HAS_BUILTIN(__remove_pointer) && _CCCL_CUDA_COMPILER(CLANG)
# define _CCCL_BUILTIN_REMOVE_POINTER(...) __remove_pointer(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__remove_pointer)
#if _CCCL_HAS_BUILTIN(__remove_reference)
# define _CCCL_BUILTIN_REMOVE_REFERENCE_T(...) __remove_reference(__VA_ARGS__)
#elif _CCCL_HAS_BUILTIN(__remove_reference_t) && _CCCL_CUDA_COMPILER(CLANG)
# define _CCCL_BUILTIN_REMOVE_REFERENCE_T(...) __remove_reference_t(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__remove_reference_t)
#if _CCCL_COMPILER(NVRTC, <, 12, 4) // NVRTC below 12.4 fails to properly compile cuda::std::move with that
# undef _CCCL_BUILTIN_REMOVE_REFERENCE_T
#endif // _CCCL_COMPILER(NVRTC, <, 12, 4)
#if _CCCL_HAS_BUILTIN(__remove_volatile) && _CCCL_CUDA_COMPILER(CLANG)
# define _CCCL_BUILTIN_REMOVE_VOLATILE(...) __remove_volatile(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__remove_volatile)
#if _CCCL_HAS_BUILTIN(__type_pack_element)
# define _CCCL_BUILTIN_TYPE_PACK_ELEMENT(...) __type_pack_element<__VA_ARGS__>
#endif // _CCCL_HAS_BUILTIN(__type_pack_element)
#if _CCCL_HAS_BUILTIN(__is_complete_type)
# define _CCCL_BUILTIN_IS_COMPLETE_TYPE(...) __is_complete_type(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__is_complete_type)
#if _CCCL_HAS_BUILTIN(__builtin_clear_padding) \
&& (_CCCL_HOST_COMPILATION() || !(_CCCL_COMPILER(GCC) || _CCCL_COMPILER(NVHPC)))
# define _CCCL_BUILTIN_CLEAR_PADDING(...) __builtin_clear_padding(__VA_ARGS__)
#endif // _CCCL_HAS_BUILTIN(__builtin_clear_padding) && (_CCCL_HOST_COMPILATION() || !(_CCCL_COMPILER(GCC) ||
// _CCCL_COMPILER(NVHPC)))
// NVCC prior to 12.2 have trouble with pack expansion into __type_pack_element in an alias template
#if _CCCL_CUDACC_BELOW(12, 2)
# undef _CCCL_BUILTIN_TYPE_PACK_ELEMENT
#endif // _CCCL_CUDACC_BELOW(12, 2)
#if _CCCL_COMPILER(MSVC) // To use __builtin_FUNCSIG(), both MSVC and nvcc need to support it
# if _CCCL_COMPILER(MSVC, >=, 19, 35) && _CCCL_CUDACC_AT_LEAST(12, 3)
# define _CCCL_BUILTIN_PRETTY_FUNCTION() __builtin_FUNCSIG()
# else // ^^^ _CCCL_COMPILER(MSVC, >=, 19, 35) ^^^ / vvv _CCCL_COMPILER(MSVC, <, 19, 35) vvv
# define _CCCL_BUILTIN_PRETTY_FUNCTION() __FUNCSIG__
# define _CCCL_BROKEN_MSVC_FUNCSIG
# endif // _CCCL_COMPILER(MSVC, <, 19, 35)
#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
# define _CCCL_BUILTIN_PRETTY_FUNCTION() __PRETTY_FUNCTION__
#endif // !_CCCL_COMPILER(MSVC)
// GCC's builtin_strlen isn't reliable at constexpr time
// NVRTC does not expose builtin_strlen
#if !_CCCL_COMPILER(GCC) && !_CCCL_COMPILER(NVRTC)
# define _CCCL_BUILTIN_STRLEN(...) __builtin_strlen(__VA_ARGS__)
#endif
// The new __nv_atomic builtins are available when __CUDACC_DEVICE_ATOMIC_BUILTINS__ is defined
#if defined(__CUDACC_DEVICE_ATOMIC_BUILTINS__) && _CCCL_PTX_ARCH() >= 600 && !_CCCL_COMPILER(MSVC)
# define _CCCL_HAS_NV_ATOMIC_BUILTINS() 1
#else // ^^^ has intrinsics ^^^ / vvv no intrinsics
# define _CCCL_HAS_NV_ATOMIC_BUILTINS() 0
#endif // no intrinsics
#endif // __CCCL_BUILTIN_H

238
cccl_upstream/libcudacxx/include/cuda/std/__cccl/compiler.h Normal file
View File

@@ -0,0 +1,238 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_COMPILER_H
#define __CCCL_COMPILER_H
#include <cuda/std/__cccl/preprocessor.h>
// Utility to compare version numbers. To use:
// 1) Define a macro that makes a pair of (major, minor) numbers:
// #define MYPRODUCT_MAKE_VERSION(_MAJOR, _MINOR) (_MAJOR * 100 + _MINOR)
// 2) Define a macro that you will use to compare versions, e.g.:
// #define MYPRODUCT(...) _CCCL_VERSION_COMPARE(MYPRODUCT, MYPRODUCT_##__VA_ARGS__)
// Signatures:
// MYPRODUCT(_PROD) - is the product _PROD version non-zero?
// MYPRODUCT(_PROD, _OP, _MAJOR) - compare the product _PROD major version to _MAJOR using operator _OP
// MYPRODUCT(_PROD, _OP, _MAJOR, _MINOR) - compare the product _PROD version to _MAJOR._MINOR using operator _OP
// 3) Define the product version macros as a function-like macro that returns the version number or
// _CCCL_VERSION_INVALID() if the version cannot be determined, e. g.:
// #define MYPRODUCT_<_PROD>() (1, 2)
// or
// #define MYPRODUCT_<_PROD>() _CCCL_VERSION_INVALID()
#define _CCCL_VERSION_MAJOR_(_MAJOR, _MINOR) _MAJOR
#define _CCCL_VERSION_MAJOR(_PAIR) _CCCL_VERSION_MAJOR_ _PAIR
#define _CCCL_VERSION_INVALID() (-1, -1)
#define _CCCL_MAKE_VERSION(_PREFIX, _PAIR) (_CCCL_PP_EVAL(_CCCL_PP_CAT(_PREFIX, MAKE_VERSION), _CCCL_PP_EXPAND _PAIR))
#define _CCCL_VERSION_IS_INVALID(_PAIR) (_CCCL_VERSION_MAJOR(_PAIR) == _CCCL_VERSION_MAJOR(_CCCL_VERSION_INVALID()))
#define _CCCL_VERSION_COMPARE_1(_PREFIX, _VER) (!_CCCL_VERSION_IS_INVALID(_VER()))
#define _CCCL_VERSION_COMPARE_3(_PREFIX, _VER, _OP, _MAJOR) \
(!_CCCL_VERSION_IS_INVALID(_VER()) && (_CCCL_VERSION_MAJOR(_VER()) _OP _MAJOR))
#define _CCCL_VERSION_COMPARE_4(_PREFIX, _VER, _OP, _MAJOR, _MINOR) \
(!_CCCL_VERSION_IS_INVALID(_VER()) \
&& (_CCCL_MAKE_VERSION(_PREFIX, _VER()) _OP _CCCL_MAKE_VERSION(_PREFIX, (_MAJOR, _MINOR))))
#define _CCCL_VERSION_SELECT_COUNT(_ARG1, _ARG2, _ARG3, _ARG4, _ARG5, ...) _ARG5
#define _CCCL_VERSION_SELECT2(_ARGS) _CCCL_VERSION_SELECT_COUNT _ARGS
// MSVC traditonal preprocessor requires an extra level of indirection
#define _CCCL_VERSION_SELECT(...) \
_CCCL_VERSION_SELECT2( \
(__VA_ARGS__, \
_CCCL_VERSION_COMPARE_4, \
_CCCL_VERSION_COMPARE_3, \
_CCCL_VERSION_COMPARE_BAD_ARG_COUNT, \
_CCCL_VERSION_COMPARE_1, \
_CCCL_VERSION_COMPARE_BAD_ARG_COUNT))
#define _CCCL_VERSION_COMPARE(_PREFIX, ...) _CCCL_VERSION_SELECT(__VA_ARGS__)(_PREFIX, __VA_ARGS__)
#define _CCCL_COMPILER_MAKE_VERSION(_MAJOR, _MINOR) ((_MAJOR) * 100 + (_MINOR))
#define _CCCL_COMPILER(...) _CCCL_VERSION_COMPARE(_CCCL_COMPILER_, _CCCL_COMPILER_##__VA_ARGS__)
#define _CCCL_COMPILER_NVHPC() _CCCL_VERSION_INVALID()
#define _CCCL_COMPILER_CLANG() _CCCL_VERSION_INVALID()
#define _CCCL_COMPILER_GCC() _CCCL_VERSION_INVALID()
#define _CCCL_COMPILER_MSVC() _CCCL_VERSION_INVALID()
#define _CCCL_COMPILER_MSVC2019() _CCCL_VERSION_INVALID()
#define _CCCL_COMPILER_MSVC2022() _CCCL_VERSION_INVALID()
#define _CCCL_COMPILER_MSVC2026() _CCCL_VERSION_INVALID()
#define _CCCL_COMPILER_NVRTC() _CCCL_VERSION_INVALID()
// Determine the host compiler and its version
#if defined(__INTEL_COMPILER)
# ifndef CCCL_IGNORE_DEPRECATED_COMPILER
# warning \
"The Intel C++ Compiler Classic (icc/icpc) is not supported by CCCL. Define CCCL_IGNORE_DEPRECATED_COMPILER to suppress this message."
# endif // !CCCL_IGNORE_DEPRECATED_COMPILER
#elif defined(__NVCOMPILER)
# undef _CCCL_COMPILER_NVHPC
# define _CCCL_COMPILER_NVHPC() (__NVCOMPILER_MAJOR__, __NVCOMPILER_MINOR__)
#elif defined(__clang__)
# undef _CCCL_COMPILER_CLANG
# define _CCCL_COMPILER_CLANG() (__clang_major__, __clang_minor__)
#elif defined(__GNUC__)
# undef _CCCL_COMPILER_GCC
# define _CCCL_COMPILER_GCC() (__GNUC__, __GNUC_MINOR__)
#elif defined(_MSC_VER)
// see https://learn.microsoft.com/en-us/cpp/overview/compiler-versions?view=msvc-180#version-macros
# undef _CCCL_COMPILER_MSVC
# define _CCCL_COMPILER_MSVC() (_MSC_VER / 100, _MSC_VER % 100)
# if _CCCL_COMPILER(MSVC, <, 19, 20)
# ifndef CCCL_IGNORE_DEPRECATED_COMPILER
# error \
"Visual Studio 2017 (MSC_VER < 1920) and older are not supported by CCCL. Define CCCL_IGNORE_DEPRECATED_COMPILER to suppress this error."
# endif
# endif // _CCCL_COMPILER(MSVC, <, 19, 20)
# if _CCCL_COMPILER(MSVC, >=, 19, 20) && _CCCL_COMPILER(MSVC, <, 19, 30)
# undef _CCCL_COMPILER_MSVC2019
# define _CCCL_COMPILER_MSVC2019() _CCCL_COMPILER_MSVC()
# endif // _CCCL_COMPILER(MSVC, >=, 19, 20) && _CCCL_COMPILER(MSVC, <, 19, 30)
# if _CCCL_COMPILER(MSVC, >=, 19, 30) && _CCCL_COMPILER(MSVC, <, 19, 50)
# undef _CCCL_COMPILER_MSVC2022
# define _CCCL_COMPILER_MSVC2022() _CCCL_COMPILER_MSVC()
# endif // _CCCL_COMPILER(MSVC, >=, 19, 30) && _CCCL_COMPILER(MSVC, <, 19, 50)
# if _CCCL_COMPILER(MSVC, >=, 19, 50)
# undef _CCCL_COMPILER_MSVC2026
# define _CCCL_COMPILER_MSVC2026() _CCCL_COMPILER_MSVC()
# endif // _CCCL_COMPILER(MSVC, >=, 19, 45)
#elif defined(__CUDACC_RTC__)
# undef _CCCL_COMPILER_NVRTC
# define _CCCL_COMPILER_NVRTC() (__CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__)
#endif
#if defined(__CUDACC__) || defined(_NVHPC_CUDA)
# define _CCCL_CUDA_COMPILATION() 1
#else // ^^^ compiling .cu file ^^^ / vvv not compiling .cu file vvv
# define _CCCL_CUDA_COMPILATION() 0
#endif // ^^^ not compiling .cu file ^^^
// The CUDA compiler version shares the implementation with the C++ compiler
#define _CCCL_CUDA_COMPILER_MAKE_VERSION(_MAJOR, _MINOR) _CCCL_COMPILER_MAKE_VERSION(_MAJOR, _MINOR)
#define _CCCL_CUDA_COMPILER(...) _CCCL_VERSION_COMPARE(_CCCL_CUDA_COMPILER_, _CCCL_CUDA_COMPILER_##__VA_ARGS__)
#define _CCCL_CUDA_COMPILER_NVCC() _CCCL_VERSION_INVALID()
#define _CCCL_CUDA_COMPILER_NVHPC() _CCCL_VERSION_INVALID()
#define _CCCL_CUDA_COMPILER_CLANG() _CCCL_VERSION_INVALID()
#define _CCCL_CUDA_COMPILER_NVRTC() _CCCL_VERSION_INVALID()
// Determine the cuda compiler
#if _CCCL_CUDA_COMPILATION()
# if defined(__NVCC__)
# undef _CCCL_CUDA_COMPILER_NVCC
# define _CCCL_CUDA_COMPILER_NVCC() (__CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__)
# elif defined(_NVHPC_CUDA)
# undef _CCCL_CUDA_COMPILER_NVHPC
# define _CCCL_CUDA_COMPILER_NVHPC() _CCCL_COMPILER_NVHPC()
# elif defined(__CUDA__) && _CCCL_COMPILER(CLANG)
# undef _CCCL_CUDA_COMPILER_CLANG
# define _CCCL_CUDA_COMPILER_CLANG() _CCCL_COMPILER_CLANG()
# elif _CCCL_COMPILER(NVRTC)
# undef _CCCL_CUDA_COMPILER_NVRTC
# define _CCCL_CUDA_COMPILER_NVRTC() _CCCL_COMPILER_NVRTC()
# endif // ^^^ _CCCL_COMPILER(NVRTC) ^^^
#endif // _CCCL_CUDA_COMPILATION()
// Determine if we are compiling host code, this includes both CUDA and C++ compilation
// nvc++ does not define __CUDA_ARCH__, but it compiles both host and device code at the same time
#if !defined(__CUDA_ARCH__)
# define _CCCL_HOST_COMPILATION() 1
#else // ^^^ compiling host code ^^^ / vvv not compiling host code vvv
# define _CCCL_HOST_COMPILATION() 0
#endif // ^^^ not compiling host code ^^^
#if (_CCCL_CUDA_COMPILATION() && defined(__CUDA_ARCH__)) || _CCCL_CUDA_COMPILER(NVHPC)
# define _CCCL_DEVICE_COMPILATION() 1
#else // ^^^ compiling device code ^^^ / vvv not compiling device code vvv
# define _CCCL_DEVICE_COMPILATION() 0
#endif // ^^^ not compiling device code ^^^
#if defined(__CUDACC_TILE__) && _CCCL_CUDA_COMPILER(NVCC, >, 13, 3)
# define _CCCL_TILE_COMPILATION() 1
#else // ^^^ compiling .cu file in tile mode ^^^ / vvv not compiling in tile mode vvv
# define _CCCL_TILE_COMPILATION() 0
#endif // ^^^ not compiling .cu file ^^^
#define _CCCL_CUDACC_MAKE_VERSION(_MAJOR, _MINOR) ((_MAJOR) * 1000 + (_MINOR) * 10)
// clang-cuda does not define __CUDACC_VER_MAJOR__ and friends. They are instead retrieved from the CUDA_VERSION macro
// defined in "cuda.h". clang-cuda automatically pre-includes "__clang_cuda_runtime_wrapper.h" which includes "cuda.h"
#if _CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(NVHPC) || _CCCL_CUDA_COMPILER(NVRTC)
# define _CCCL_CUDACC() (__CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__)
#elif _CCCL_CUDA_COMPILER(CLANG)
# define _CCCL_CUDACC() (CUDA_VERSION / 1000, (CUDA_VERSION % 1000) / 10)
#endif // ^^^ has cuda compiler ^^^
#if !defined(_CCCL_CUDACC) || !_CCCL_CUDA_COMPILATION()
# undef _CCCL_CUDACC
# define _CCCL_CUDACC() _CCCL_VERSION_INVALID()
#endif // !_CCCL_CUDACC || !_CCCL_CUDA_COMPILATION()
#define _CCCL_CUDACC_EQUAL(...) _CCCL_VERSION_COMPARE(_CCCL_CUDACC_, _CCCL_CUDACC, ==, __VA_ARGS__)
#define _CCCL_CUDACC_BELOW(...) _CCCL_VERSION_COMPARE(_CCCL_CUDACC_, _CCCL_CUDACC, <, __VA_ARGS__)
#define _CCCL_CUDACC_AT_LEAST(...) _CCCL_VERSION_COMPARE(_CCCL_CUDACC_, _CCCL_CUDACC, >=, __VA_ARGS__)
#if _CCCL_CUDA_COMPILATION() && _CCCL_CUDACC_BELOW(12) && !defined(CCCL_IGNORE_DEPRECATED_CUDA_BELOW_12)
# error "CUDA versions below 12 are not supported." \
"Define CCCL_IGNORE_DEPRECATED_CUDA_BELOW_12 to suppress this message."
#endif
// Define the pragma for the host compiler
#if _CCCL_COMPILER(MSVC)
# define _CCCL_PRAGMA(_ARG) __pragma(_ARG)
#else
# define _CCCL_PRAGMA(_ARG) _Pragma(_CCCL_TO_STRING(_ARG))
#endif // _CCCL_COMPILER(MSVC)
// Define the proper object format for NVHPC and NVRTC
#if (_CCCL_COMPILER(NVHPC) && defined(__linux__)) || _CCCL_COMPILER(NVRTC)
# ifndef __ELF__
# define __ELF__
# endif // !__ELF__
#endif // _CCCL_COMPILER(NVHPC) || _CCCL_COMPILER(NVRTC)
#if _CCCL_DEVICE_COMPILATION()
# define _CCCL_PRAGMA_UNROLL(_N) _CCCL_PRAGMA(unroll _N)
# define _CCCL_PRAGMA_UNROLL_FULL() _CCCL_PRAGMA(unroll)
#elif _CCCL_COMPILER(NVHPC) || _CCCL_COMPILER(NVRTC) || _CCCL_COMPILER(CLANG)
# define _CCCL_PRAGMA_UNROLL(_N) _CCCL_PRAGMA(unroll _N)
# define _CCCL_PRAGMA_UNROLL_FULL() _CCCL_PRAGMA(unroll)
#elif _CCCL_COMPILER(GCC, >=, 8)
// gcc supports only #pragma GCC unroll, but that causes problems when compiling with nvcc. So, we use #pragma unroll
// when compiling device code, and #pragma GCC unroll when compiling host code, but we need to suppress the warning
// about the unknown pragma for nvcc.
// #pragma GCC unroll does not support full unrolling, so we use the maximum value that it supports.
# define _CCCL_PRAGMA_UNROLL(_N) \
_CCCL_BEGIN_NV_DIAG_SUPPRESS(1675) _CCCL_PRAGMA(GCC unroll _N) _CCCL_END_NV_DIAG_SUPPRESS()
# define _CCCL_PRAGMA_UNROLL_FULL() _CCCL_PRAGMA_UNROLL(65534)
#else // ^^^ has pragma unroll support ^^^ / vvv no pragma unroll support vvv
# define _CCCL_PRAGMA_UNROLL(_N)
# define _CCCL_PRAGMA_UNROLL_FULL()
#endif // ^^^ no pragma unroll support ^^^
#define _CCCL_PRAGMA_NOUNROLL() _CCCL_PRAGMA_UNROLL(1)
#if _CCCL_COMPILER(MSVC)
# define _CCCL_WARNING(_MSG) _CCCL_PRAGMA(message(__FILE__ ":" _CCCL_TO_STRING(__LINE__) ": warning: " _MSG))
#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
# define _CCCL_WARNING(_MSG) _CCCL_PRAGMA(GCC warning _MSG)
#endif // !_CCCL_COMPILER(MSVC)
// Freestanding environment detection
// NVRTC is treated as freestanding since it has no access to the host standard library
#if defined(_CCCL_ENABLE_FREESTANDING) || _CCCL_COMPILER(NVRTC)
# define _CCCL_FREESTANDING() 1
# define _CCCL_HOSTED() 0
# define _CCCL_HOSTJIT() (!_CCCL_COMPILER(NVRTC))
# define _CCCL_NO_TYPEID
#else // ^^^ _CCCL_ENABLE_FREESTANDING || _CCCL_COMPILER(NVRTC) ^^^ / vvv Hosted environment vvv
# define _CCCL_FREESTANDING() 0
# define _CCCL_HOSTED() 1
# define _CCCL_HOSTJIT() 0
#endif // Hosted environment
#endif // __CCCL_COMPILER_H

118
cccl_upstream/libcudacxx/include/cuda/std/__cccl/cuda_capabilities.h Normal file
View File

@@ -0,0 +1,118 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_CUDA_CAPABILITIES
#define __CCCL_CUDA_CAPABILITIES
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__cccl/cuda_toolkit.h>
#include <nv/target>
/// In device code, _CCCL_PTX_ARCH() expands to the PTX version for which we are compiling.
/// In host code, _CCCL_PTX_ARCH()'s value is implementation defined.
#if !defined(__CUDA_ARCH__)
# define _CCCL_PTX_ARCH() 0
#else
# define _CCCL_PTX_ARCH() __CUDA_ARCH__
#endif
#ifdef _CCCL_DOXYGEN_INVOKED // Only parse this during doxygen passes:
//! When this macro is defined, Programmatic Dependent Launch (PDL) is disabled across CCCL
# define CCCL_DISABLE_PDL
#endif // _CCCL_DOXYGEN_INVOKED
#ifdef CCCL_DISABLE_PDL
# define _CCCL_HAS_PDL() 0
#else // CCCL_DISABLE_PDL
# define _CCCL_HAS_PDL() 1
#endif // CCCL_DISABLE_PDL
#if _CCCL_HAS_PDL()
// Waits for the previous kernel to complete (when it reaches its final membar). Should be put before the first global
// memory access in a kernel.
# define _CCCL_PDL_GRID_DEPENDENCY_SYNC() NV_IF_TARGET(NV_PROVIDES_SM_90, ::cudaGridDependencySynchronize();)
// Allows the subsequent kernel in the same stream to launch. Can be put anywhere in a kernel.
// Heuristic(ahendriksen): put it after the last load.
# define _CCCL_PDL_TRIGGER_NEXT_LAUNCH() NV_IF_TARGET(NV_PROVIDES_SM_90, ::cudaTriggerProgrammaticLaunchCompletion();)
#else // _CCCL_HAS_PDL()
# define _CCCL_PDL_GRID_DEPENDENCY_SYNC()
# define _CCCL_PDL_TRIGGER_NEXT_LAUNCH()
#endif // _CCCL_HAS_PDL()
// Check whether the relocatable device code (RDC) is being generated.
#if defined(__CUDACC_RDC__) || defined(__CLANG_RDC__) || defined(_NVHPC_RDC)
# define _CCCL_HAS_RDC() 1
#else // ^^^ has RDC ^^^ / vvv no RDC vvv
# define _CCCL_HAS_RDC() 0
#endif // ^^^ no RDC ^^^
// Check whether extensible whole program is being compiled.
#if defined(__CUDACC_EWP__)
# define _CCCL_HAS_EWP() 1
#else // ^^^ has EWP ^^^ / vvv no EWP vvv
# define _CCCL_HAS_EWP() 0
#endif // ^^^ no EWP ^^^
// Control whether device runtime APIs can be used, because they require libcudadevrt to be linked. Defaults to true
// when RDC or EWP are enabled. Can be disabled by defining CCCL_DISABLE_DEVICE_RUNTIME.
#if (_CCCL_HAS_RDC() || _CCCL_HAS_EWP()) && !defined(CCCL_DISABLE_DEVICE_RUNTIME)
# define _CCCL_HAS_DEVICE_RUNTIME() 1
#else // ^^^ has device runtime ^^^ / vvv no device runtime vvv
# define _CCCL_HAS_DEVICE_RUNTIME() 0
#endif // ^^^ no device runtime ^^^
// Some functions can be called from host or device code and launch kernels inside. Thus, they use CUDA Dynamic
// Parallelism (CDP) and require compiling with Relocatable Device Code (RDC) or extensible whole program (EWP) and link
// with device runtime library. CDP is unsupported with clang-cuda below 22.
// TODO(bgruber): remove CUB_DISABLE_CDP in CCCL 4.0
#if _CCCL_HAS_DEVICE_RUNTIME() && !defined(CCCL_DISABLE_CDP) && !defined(CUB_DISABLE_CDP) \
&& !_CCCL_CUDA_COMPILER(CLANG, <, 22)
// We have CDP, so host and device APIs can call kernels
# define _CCCL_HAS_CDP() 1
#else // ^^^ has CDP ^^^ / vvv no CDP vvv
// We don't have CDP, only host APIs can call kernels
# define _CCCL_HAS_CDP() 0
#endif // ^^^ no CDP ^^^
// When RDC is enabled, __launch_bounds__ cannot be used reliably. See #902.
#if !_CCCL_HAS_RDC() && !defined(CCCL_DISABLE_LAUNCH_BOUNDS)
# define _CCCL_LAUNCH_BOUNDS(...) __launch_bounds__(__VA_ARGS__)
#else // ^^^ has launch bounds attribute ^^^ / vvv no launch bounds attribute vvv
# define _CCCL_LAUNCH_BOUNDS(...)
#endif // ^^^ no launch bounds attribute ^^^
// __block_size__ attribute is available for nvcc and nvrtc 12.9+ for hopper+ architectures. For older nvcc and nvrtc,
// we can fallback to __cluster_dims__ attribute only specifying the ncta per cluster.
// This attribute should be used only for cluster launches.
#if (_CCCL_CUDA_COMPILER(NVCC, >=, 12, 9) || _CCCL_CUDA_COMPILER(NVRTC, >=, 12, 9)) && _CCCL_PTX_ARCH() >= 900
# define _CCCL_BLOCK_SIZE(_NTID, _NCTA_PER_CLUSTER) __block_size__(_NTID, _NCTA_PER_CLUSTER)
#elif (_CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(NVRTC)) && _CCCL_PTX_ARCH() >= 900
# define _CCCL_BLOCK_SIZE(_NTID, _NCTA_PER_CLUSTER) __cluster_dims__ _NCTA_PER_CLUSTER
#else // ^^ has __block_size__ attribute ^^^ / vvv no __block_size__ attribute vvv
# define _CCCL_BLOCK_SIZE(_NTID, _NCTA_PER_CLUSTER)
#endif // ^^^ no __block_size__ attribute ^^^
#if _CCCL_HAS_CDP()
# ifdef CUDA_FORCE_CDP1_IF_SUPPORTED
# error "CUDA Dynamic Parallelism 1 is no longer supported. Please undefine CUDA_FORCE_CDP1_IF_SUPPORTED."
# endif // CUDA_FORCE_CDP1_IF_SUPPORTED
#endif // _CCCL_HAS_CDP()
#endif // __CCCL_CUDA_CAPABILITIES

56
cccl_upstream/libcudacxx/include/cuda/std/__cccl/cuda_toolkit.h Normal file
View File

@@ -0,0 +1,56 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_CUDA_TOOLKIT_H
#define __CCCL_CUDA_TOOLKIT_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_CUDA_COMPILATION() || __has_include(<cuda_runtime_api.h>)
# define _CCCL_HAS_CTK() 1
#else // ^^^ has cuda toolkit ^^^ / vvv no cuda toolkit vvv
# define _CCCL_HAS_CTK() 0
#endif // ^^^ no cuda toolkit ^^^
// CUDA compilers preinclude cuda_runtime.h, so we need to include it here to get the CUDART_VERSION macro
#if _CCCL_HAS_CTK() && !_CCCL_CUDA_COMPILATION()
# include <cuda_runtime_api.h>
#endif // _CCCL_HAS_CTK() && !_CCCL_CUDA_COMPILATION()
// Check compatibility of the CUDA compiler and CUDA toolkit headers
// Some users might want to use a newer version of the CTK than the compiler ships. Enable that on their own peril
#ifndef CCCL_DISABLE_CTK_COMPATIBILITY_CHECK
# if _CCCL_CUDA_COMPILATION()
# if !_CCCL_CUDACC_EQUAL((CUDART_VERSION / 1000), (CUDART_VERSION % 1000) / 10)
# error "CUDA compiler and CUDA toolkit headers are incompatible, please check your include paths"
# endif // !_CCCL_CUDACC_EQUAL((CUDART_VERSION / 1000), (CUDART_VERSION % 1000) / 10)
# endif // _CCCL_CUDA_COMPILATION()
#endif // CCCL_DISABLE_CTK_COMPATIBILITY_CHECK
#if _CCCL_HAS_CTK()
# define _CCCL_CTK() (CUDART_VERSION / 1000, (CUDART_VERSION % 1000) / 10)
#else // ^^^ has cuda toolkit ^^^ / vvv no cuda toolkit vvv
# define _CCCL_CTK() _CCCL_VERSION_INVALID()
#endif // ^^^ no cuda toolkit ^^^
#define _CCCL_CTK_MAKE_VERSION(_MAJOR, _MINOR) ((_MAJOR) * 1000 + (_MINOR) * 10)
#define _CCCL_CTK_BELOW(...) _CCCL_VERSION_COMPARE(_CCCL_CTK_, _CCCL_CTK, <, __VA_ARGS__)
#define _CCCL_CTK_AT_LEAST(...) _CCCL_VERSION_COMPARE(_CCCL_CTK_, _CCCL_CTK, >=, __VA_ARGS__)
#endif // __CCCL_CUDA_TOOLKIT_H

87
cccl_upstream/libcudacxx/include/cuda/std/__cccl/deprecated.h Normal file
View File

@@ -0,0 +1,87 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_DEPRECATED_H
#define __CCCL_DEPRECATED_H
#include <cuda/std/__cccl/attributes.h>
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/dialect.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
// Check for deprecation opt outs
#if defined(LIBCUDACXX_IGNORE_DEPRECATED_CPP_DIALECT)
# if !defined(CCCL_IGNORE_DEPRECATED_CPP_DIALECT)
# define CCCL_IGNORE_DEPRECATED_CPP_DIALECT
# endif
#endif // suppress all dialect deprecation warnings
#if defined(LIBCUDACXX_IGNORE_DEPRECATED_CPP_14) || defined(CCCL_IGNORE_DEPRECATED_CPP_DIALECT)
# if !defined(CCCL_IGNORE_DEPRECATED_CPP_14)
# define CCCL_IGNORE_DEPRECATED_CPP_14
# endif
#endif // suppress all c++14 dialect deprecation warnings
#if defined(LIBCUDACXX_IGNORE_DEPRECATED_CPP_11) || defined(CCCL_IGNORE_DEPRECATED_CPP_DIALECT) \
|| defined(CCCL_IGNORE_DEPRECATED_CPP_14)
# if !defined(CCCL_IGNORE_DEPRECATED_CPP_11)
# define CCCL_IGNORE_DEPRECATED_CPP_11
# endif
#endif // suppress all c++11 dialect deprecation warnings
#if defined(LIBCUDACXX_IGNORE_DEPRECATED_COMPILER) || defined(THRUST_IGNORE_DEPRECATED_COMPILER) \
|| defined(CUB_IGNORE_DEPRECATED_COMPILER) || defined(CCCL_IGNORE_DEPRECATED_CPP_DIALECT) \
|| defined(CCCL_IGNORE_DEPRECATED_CPP_14) || defined(CCCL_IGNORE_DEPRECATED_CPP_11)
# if !defined(CCCL_IGNORE_DEPRECATED_COMPILER)
# define CCCL_IGNORE_DEPRECATED_COMPILER
# endif
#endif // suppress all compiler deprecation warnings
#if defined(LIBCUDACXX_IGNORE_DEPRECATED_API) || defined(THRUST_IGNORE_DEPRECATED_API) \
|| defined(CUB_IGNORE_DEPRECATED_API)
# if !defined(CCCL_IGNORE_DEPRECATED_API)
# define CCCL_IGNORE_DEPRECATED_API
# endif
#endif // suppress all API deprecation warnings
#if defined(CCCL_IGNORE_DEPRECATED_API) || defined(_LIBCUDACXX_DISABLE_DEPRECATION_WARNINGS)
//! deprecated [Since 2.8]
# define CCCL_DEPRECATED
//! deprecated [Since 2.8]
# define CCCL_DEPRECATED_BECAUSE(MSG)
#elif _CCCL_HAS_ATTRIBUTE(deprecated)
//! deprecated [Since 2.8]
# define CCCL_DEPRECATED __attribute__((deprecated))
//! deprecated [Since 2.8]
# define CCCL_DEPRECATED_BECAUSE(MSG) __attribute__((deprecated(MSG)))
#else // ^^^ attribute deprecated ^^^ / vvv standard deprecated attribute vvv
//! deprecated [Since 2.8]
# define CCCL_DEPRECATED [[deprecated]]
//! deprecated [Since 2.8]
# define CCCL_DEPRECATED_BECAUSE(MSG) [[deprecated(MSG)]]
#endif // ^^^ standard deprecated attribute ^^^
#if _CCCL_STD_VER >= 2020
# define _CCCL_DEPRECATED_IN_CXX20 CCCL_DEPRECATED
#else // ^^^ _CCCL_STD_VER >= 2020 ^^^ / vvv _CCCL_STD_VER < 2020 vvv
# define _CCCL_DEPRECATED_IN_CXX20
#endif // ^^^ _CCCL_STD_VER < 2020 ^^^
#if _CCCL_STD_VER >= 2023
# define _CCCL_DEPRECATED_IN_CXX23 CCCL_DEPRECATED
#else // ^^^ _CCCL_STD_VER >= 2023 ^^^ / vvv _CCCL_STD_VER < 2023 vvv
# define _CCCL_DEPRECATED_IN_CXX23
#endif // ^^^ _CCCL_STD_VER < 2023 ^^^
#endif // __CCCL_DEPRECATED_H

145
cccl_upstream/libcudacxx/include/cuda/std/__cccl/diagnostic.h Normal file
View File

@@ -0,0 +1,145 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_DIAGNOSTIC_H
#define __CCCL_DIAGNOSTIC_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
// Enable us to selectively silence host compiler warnings
#if _CCCL_COMPILER(CLANG)
# define _CCCL_DIAG_PUSH _CCCL_PRAGMA(clang diagnostic push)
# define _CCCL_DIAG_POP _CCCL_PRAGMA(clang diagnostic pop)
# define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING) _CCCL_PRAGMA(clang diagnostic ignored _WARNING)
# define _CCCL_DIAG_SUPPRESS_GCC(_WARNING)
# define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING)
# define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING)
#elif _CCCL_COMPILER(GCC)
# define _CCCL_DIAG_PUSH _CCCL_PRAGMA(GCC diagnostic push)
# define _CCCL_DIAG_POP _CCCL_PRAGMA(GCC diagnostic pop)
# define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING)
# define _CCCL_DIAG_SUPPRESS_GCC(_WARNING) _CCCL_PRAGMA(GCC diagnostic ignored _WARNING)
# define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING)
# define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING)
#elif _CCCL_COMPILER(NVHPC)
# define _CCCL_DIAG_PUSH _CCCL_PRAGMA(diagnostic push)
# define _CCCL_DIAG_POP _CCCL_PRAGMA(diagnostic pop)
# define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING)
# define _CCCL_DIAG_SUPPRESS_GCC(_WARNING)
# define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING) _CCCL_PRAGMA(diag_suppress _WARNING)
# define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING)
#elif _CCCL_COMPILER(MSVC)
# define _CCCL_DIAG_PUSH _CCCL_PRAGMA(warning(push))
# define _CCCL_DIAG_POP _CCCL_PRAGMA(warning(pop))
# define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING)
# define _CCCL_DIAG_SUPPRESS_GCC(_WARNING)
# define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING)
# define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING) _CCCL_PRAGMA(warning(disable : _WARNING))
#else
# define _CCCL_DIAG_PUSH
# define _CCCL_DIAG_POP
# define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING)
# define _CCCL_DIAG_SUPPRESS_GCC(_WARNING)
# define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING)
# define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING)
#endif
// Enable us to selectively silence cuda compiler warnings
#if _CCCL_CUDA_COMPILER(NVCC) || _CCCL_COMPILER(NVRTC)
# if defined(__NVCC_DIAG_PRAGMA_SUPPORT__)
# define _CCCL_NV_DIAG_PUSH() _CCCL_PRAGMA(nv_diagnostic push)
# define _CCCL_NV_DIAG_POP() _CCCL_PRAGMA(nv_diagnostic pop)
# define _CCCL_DIAG_SUPPRESS_NVCC(_WARNING) _CCCL_PRAGMA(nv_diag_suppress _WARNING)
# define _CCCL_BEGIN_NV_DIAG_SUPPRESS(...) \
_CCCL_NV_DIAG_PUSH() _CCCL_PP_FOR_EACH(_CCCL_DIAG_SUPPRESS_NVCC, __VA_ARGS__)
# define _CCCL_END_NV_DIAG_SUPPRESS() _CCCL_NV_DIAG_POP()
# else // ^^^ __NVCC_DIAG_PRAGMA_SUPPORT__ ^^^ / vvv !__NVCC_DIAG_PRAGMA_SUPPORT__ vvv
# define _CCCL_NV_DIAG_PUSH() _CCCL_PRAGMA(diagnostic push)
# define _CCCL_NV_DIAG_POP() _CCCL_PRAGMA(diagnostic pop)
# define _CCCL_DIAG_SUPPRESS_NVCC(_WARNING) _CCCL_PRAGMA(diag_suppress _WARNING)
# define _CCCL_BEGIN_NV_DIAG_SUPPRESS(...) \
_CCCL_NV_DIAG_PUSH() _CCCL_PP_FOR_EACH(_CCCL_DIAG_SUPPRESS_NVCC, __VA_ARGS__)
# define _CCCL_END_NV_DIAG_SUPPRESS() _CCCL_NV_DIAG_POP()
# endif // !__NVCC_DIAG_PRAGMA_SUPPORT__
#else // ^^^ _CCCL_CUDA_COMPILER(NVCC) ^^^ / vvv !_CCCL_CUDA_COMPILER(NVCC) vvv
# define _CCCL_NV_DIAG_PUSH()
# define _CCCL_NV_DIAG_POP()
# define _CCCL_DIAG_SUPPRESS_NVCC(_WARNING)
# define _CCCL_BEGIN_NV_DIAG_SUPPRESS(...)
# define _CCCL_END_NV_DIAG_SUPPRESS()
#endif // !_CCCL_CUDA_COMPILER(NVCC)
// Convenient shortcuts to silence common warnings
#if _CCCL_COMPILER(CLANG)
# define _CCCL_SUPPRESS_DEPRECATED_PUSH \
_CCCL_DIAG_PUSH \
_CCCL_DIAG_SUPPRESS_CLANG("-Wdeprecated") \
_CCCL_DIAG_SUPPRESS_CLANG("-Wdeprecated-declarations") \
_CCCL_BEGIN_NV_DIAG_SUPPRESS(1444, 20199)
# define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
# define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP() _CCCL_DIAG_POP
#elif _CCCL_COMPILER(GCC)
# define _CCCL_SUPPRESS_DEPRECATED_PUSH \
_CCCL_DIAG_PUSH \
_CCCL_DIAG_SUPPRESS_GCC("-Wdeprecated") \
_CCCL_DIAG_SUPPRESS_GCC("-Wdeprecated-declarations") \
_CCCL_BEGIN_NV_DIAG_SUPPRESS(1444, 20199)
# define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
# define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP() _CCCL_DIAG_POP
#elif _CCCL_COMPILER(NVHPC)
# define _CCCL_SUPPRESS_DEPRECATED_PUSH \
_CCCL_DIAG_PUSH \
_CCCL_DIAG_SUPPRESS_NVHPC(deprecated_entity) \
_CCCL_DIAG_SUPPRESS_NVHPC(deprecated_entity_with_custom_message) \
_CCCL_BEGIN_NV_DIAG_SUPPRESS(1444, 20199)
# define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
# define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP() _CCCL_DIAG_POP
#elif _CCCL_COMPILER(MSVC)
# define _CCCL_SUPPRESS_DEPRECATED_PUSH \
_CCCL_DIAG_PUSH \
_CCCL_DIAG_SUPPRESS_MSVC(4996) \
_CCCL_BEGIN_NV_DIAG_SUPPRESS(1444)
# define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
# define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP() _CCCL_DIAG_POP
#elif _CCCL_COMPILER(NVRTC)
# if _CCCL_COMPILER(NVRTC, >=, 13, 3) && defined(__NVCC_DIAG_PRAGMA_SUPPORT__)
# define _CCCL_SUPPRESS_DEPRECATED_PUSH _CCCL_NV_DIAG_PUSH()
// NVRTC 13.3 does not honor nv_diag_suppress when it is emitted in the same macro expansion as
// nv_diagnostic push. Keep the suppression in a separate source-level macro invocation.
// See https://github.com/NVIDIA/cccl/issues/9170 and nvbug 6239043.
# define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG _Pragma("nv_diag_suppress 1444,20199")
# else // ^^^ NVRTC >= 13.3 with __NVCC_DIAG_PRAGMA_SUPPORT__ ^^^
# define _CCCL_SUPPRESS_DEPRECATED_PUSH _CCCL_BEGIN_NV_DIAG_SUPPRESS(1444, 20199)
# define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
# endif // ^^^ NVRTC >= 13.3 with __NVCC_DIAG_PRAGMA_SUPPORT__ ^^^
# define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP()
#else // unknown compiler
# define _CCCL_SUPPRESS_DEPRECATED_PUSH
# define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
# define _CCCL_SUPPRESS_DEPRECATED_POP
#endif // unknown compiler
#if _CCCL_COMPILER(MSVC)
# define _CCCL_HAS_PRAGMA_MSVC_WARNING
# if !defined(_LIBCUDACXX_DISABLE_PRAGMA_MSVC_WARNING)
# define _CCCL_USE_PRAGMA_MSVC_WARNING
# endif // !_LIBCUDACXX_DISABLE_PRAGMA_MSVC_WARNING
#endif // !_CCCL_COMPILER(MSVC)
#endif // __CCCL_DIAGNOSTIC_H

230
cccl_upstream/libcudacxx/include/cuda/std/__cccl/dialect.h Normal file
View File

@@ -0,0 +1,230 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_DIALECT_H
#define __CCCL_DIALECT_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__cccl/builtin.h>
#include <cuda/std/__cccl/host_std_lib.h>
///////////////////////////////////////////////////////////////////////////////
// Determine the C++ standard dialect
///////////////////////////////////////////////////////////////////////////////
#if _CCCL_COMPILER(MSVC)
# if _MSVC_LANG <= 201103L
# define _CCCL_STD_VER 2011
# elif _MSVC_LANG <= 201402L
# define _CCCL_STD_VER 2014
# elif _MSVC_LANG <= 201703L
# define _CCCL_STD_VER 2017
# elif _MSVC_LANG <= 202002L
# define _CCCL_STD_VER 2020
# else
# define _CCCL_STD_VER 2023 // current year, or date of c++2b ratification
# endif
#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
# if __cplusplus <= 199711L
# define _CCCL_STD_VER 2003
# elif __cplusplus <= 201103L
# define _CCCL_STD_VER 2011
# elif __cplusplus <= 201402L
# define _CCCL_STD_VER 2014
# elif __cplusplus <= 201703L
# define _CCCL_STD_VER 2017
# elif __cplusplus <= 202002L
# define _CCCL_STD_VER 2020
# elif __cplusplus <= 202302L
# define _CCCL_STD_VER 2023
# else
# define _CCCL_STD_VER 2024 // current year, or date of c++2c ratification
# endif
#endif // !_CCCL_COMPILER(MSVC)
///////////////////////////////////////////////////////////////////////////////
// Conditionally enable constexpr per standard dialect
///////////////////////////////////////////////////////////////////////////////
#if _CCCL_STD_VER >= 2020
# define _CCCL_CONSTEXPR_CXX20 constexpr
#else // ^^^ C++20 ^^^ / vvv C++17 vvv
# define _CCCL_CONSTEXPR_CXX20
#endif // _CCCL_STD_VER <= 2017
#if _CCCL_STD_VER >= 2023
# define _CCCL_CONSTEXPR_CXX23 constexpr
#else // ^^^ C++23 ^^^ / vvv C++20 vvv
# define _CCCL_CONSTEXPR_CXX23
#endif // _CCCL_STD_VER <= 2020
///////////////////////////////////////////////////////////////////////////////
// Detect whether we can use some language features based on standard dialect
///////////////////////////////////////////////////////////////////////////////
// concepts are only available from C++20 onwards
#if _CCCL_STD_VER <= 2017 || __cpp_concepts < 201907L
# define _CCCL_HAS_CONCEPTS() 0
#else // ^^^ no concepts ^^^ / vvv has concepts vvv
# define _CCCL_HAS_CONCEPTS() 1
#endif // ^^^ has concepts ^^^
// Three way comparison is only available from C++20 onwards
#if _CCCL_STD_VER <= 2017 || __cpp_impl_three_way_comparison < 201907L
# define _CCCL_NO_THREE_WAY_COMPARISON
#endif // _CCCL_STD_VER <= 2017 || __cpp_impl_three_way_comparison < 201907L
// Some compilers turn on pack indexing in pre-C++26 code. We want to use it if it is
// available.
#if __cpp_pack_indexing >= 202311L && !_CCCL_CUDA_COMPILER(NVCC) && !_CCCL_COMPILER(CLANG, <, 20)
# define _CCCL_HAS_PACK_INDEXING() 1
#else // ^^^ has pack indexing ^^^ / vvv no pack indexing vvv
# define _CCCL_HAS_PACK_INDEXING() 0
#endif // no pack indexing
#if _CCCL_STD_VER <= 2017 || __cpp_consteval < 201811L
# define _CCCL_NO_CONSTEVAL
# define _CCCL_CONSTEVAL constexpr
#else
# define _CCCL_CONSTEVAL consteval
#endif
///////////////////////////////////////////////////////////////////////////////
// Conditionally use certain language features depending on availability
///////////////////////////////////////////////////////////////////////////////
// We need to treat host and device separately
#if _CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC)
# define _CCCL_GLOBAL_CONSTANT _CCCL_DEVICE constexpr
#else // ^^^ _CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC) ^^^ /
// vvv !_CCCL_DEVICE_COMPILATION() || _CCCL_CUDA_COMPILER(NVHPC) vvv
# define _CCCL_GLOBAL_CONSTANT inline constexpr
#endif // !_CCCL_DEVICE_COMPILATION() || _CCCL_CUDA_COMPILER(NVHPC)
#if _CCCL_STD_VER >= 2020 && __cpp_constinit >= 201907L
# define _CCCL_CONSTINIT constinit
#else // ^^^ has constinit ^^^ / vvv no constinit vvv
# define _CCCL_CONSTINIT _CCCL_REQUIRE_CONSTANT_INITIALIZATION
#endif // ^^^ no constinit ^^^
// nvcc and nvrtc don't implement multiarg operator[] even in C++23 mode
#if __cpp_multidimensional_subscript >= 202110L && !_CCCL_CUDA_COMPILER(NVCC) && !_CCCL_CUDA_COMPILER(NVRTC)
# define _CCCL_HAS_MULTIARG_OPERATOR_BRACKETS() 1
#else // ^^^ has multiarg operator[] ^^^ / vvv no multiarg operator[] vvv
# define _CCCL_HAS_MULTIARG_OPERATOR_BRACKETS() 0
#endif // ^^^ no mutiarg operator[] ^^^
// clang 16+, gcc 13+ and nvc++ 25.9+ backport the static subscript operator back to c++17.
#if __cpp_multidimensional_subscript >= 202211L \
|| ((_CCCL_COMPILER(CLANG, >=, 16) || _CCCL_COMPILER(GCC, >=, 13) \
|| (_CCCL_COMPILER(NVHPC, >=, 25, 9) && _CCCL_HOST_STD_LIB(LIBSTDCXX, >=, 12))) \
&& (!_CCCL_CUDA_COMPILATION() || _CCCL_CUDA_COMPILER(CLANG)))
# define _CCCL_HAS_STATIC_SUBSCRIPT_OPERATOR() 1
#else // ^^^ has static operator[] ^^^ / vvv no static operator[] vvv
# define _CCCL_HAS_STATIC_SUBSCRIPT_OPERATOR() 0
#endif // ^^^ no static operator[] ^^^
// nvcc 13+, clang 16+ and gcc 13+ backport the static call operator back to c++17.
#if __cpp_static_call_operator >= 202207L \
|| ((_CCCL_COMPILER(CLANG, >=, 16) || _CCCL_COMPILER(GCC, >=, 13) \
|| (_CCCL_COMPILER(NVHPC, >=, 26, 1) && _CCCL_HOST_STD_LIB(LIBSTDCXX, >=, 13))) \
&& (!_CCCL_CUDA_COMPILATION() || _CCCL_CUDA_COMPILER(NVCC, >=, 13, 0) || _CCCL_CUDA_COMPILER(CLANG)))
# define _CCCL_HAS_STATIC_CALL_OPERATOR() 1
#else // ^^^ has static operator() ^^^ / vvv no static operator() vvv
# define _CCCL_HAS_STATIC_CALL_OPERATOR() 0
#endif // ^^^ no static operator() ^^^
// if consteval requires C++23, but most compilers support it even in C++20 mode while emitting some warnings. Those are
// silenced in prologue/epilogue. nvcc is happy about using it in C++20 since 13.0, but only when compiling host code.
// nvc++ requires libstdc++ at least 12 to support if consteval.
#if _CCCL_STD_VER == 2020 \
&& (_CCCL_COMPILER(GCC, >=, 12) || _CCCL_COMPILER(CLANG) \
|| (_CCCL_COMPILER(NVHPC) && _CCCL_HOST_STD_LIB(LIBSTDCXX, >=, 12)))
# define _CCCL_HAS_IF_CONSTEVAL_IN_CXX20() 1
#else
# define _CCCL_HAS_IF_CONSTEVAL_IN_CXX20() 0
#endif
// nvcc before 13 doesn't support if consteval at all. Since 13, it accepts if consteval in host code (clang doesn't
// work) and since 13.1 it works in device code, too.
#if _CCCL_CUDA_COMPILER(NVCC, <, 13) || (_CCCL_CUDA_COMPILER(NVCC, <, 13, 1) && _CCCL_DEVICE_COMPILATION()) \
|| (_CCCL_CUDA_COMPILER(NVCC) && _CCCL_COMPILER(CLANG))
# undef _CCCL_HAS_IF_CONSTEVAL_IN_CXX20
# define _CCCL_HAS_IF_CONSTEVAL_IN_CXX20() 0
#endif // ^^^ disable if consteval in c++20 for nvcc ^^^
#if __cpp_if_consteval >= 202106L || _CCCL_HAS_IF_CONSTEVAL_IN_CXX20()
# define _CCCL_IF_CONSTEVAL if consteval
# define _CCCL_IF_CONSTEVAL_DEFAULT _CCCL_IF_CONSTEVAL
# define _CCCL_IF_NOT_CONSTEVAL if !consteval
# define _CCCL_IF_NOT_CONSTEVAL_DEFAULT _CCCL_IF_NOT_CONSTEVAL
#elif defined(_CCCL_BUILTIN_IS_CONSTANT_EVALUATED)
# if _CCCL_HOST_COMPILATION() && _CCCL_COMPILER(GCC)
# define _CCCL_BEGIN_IF_CONSTEVAL_SUPPRESS() _CCCL_DIAG_PUSH _CCCL_DIAG_SUPPRESS_GCC("-Wtautological-compare")
# define _CCCL_END_IF_CONSTEVAL_SUPPRESS() _CCCL_DIAG_POP
# else // ^^^ _CCCL_HOST_COMPILATION() && _CCCL_COMPILER(GCC) ^^^ /
// vvv !_CCCL_HOST_COMPILATION() || ! _CCCL_COMPILER(GCC) vvv
# define _CCCL_BEGIN_IF_CONSTEVAL_SUPPRESS()
# define _CCCL_END_IF_CONSTEVAL_SUPPRESS()
# endif // ^^^ !_CCCL_HOST_COMPILATION() || ! _CCCL_COMPILER(GCC) ^^^
# define _CCCL_IF_CONSTEVAL \
_CCCL_BEGIN_IF_CONSTEVAL_SUPPRESS() if (_CCCL_BUILTIN_IS_CONSTANT_EVALUATED()) _CCCL_END_IF_CONSTEVAL_SUPPRESS()
# define _CCCL_IF_CONSTEVAL_DEFAULT _CCCL_IF_CONSTEVAL
# define _CCCL_IF_NOT_CONSTEVAL \
_CCCL_BEGIN_IF_CONSTEVAL_SUPPRESS() if (!_CCCL_BUILTIN_IS_CONSTANT_EVALUATED()) _CCCL_END_IF_CONSTEVAL_SUPPRESS()
# define _CCCL_IF_NOT_CONSTEVAL_DEFAULT _CCCL_IF_NOT_CONSTEVAL
#else // ^^^ has is constant evaluated ^^^ / vvv no is constant evaluated vvv
# define _CCCL_IF_CONSTEVAL if constexpr (false)
# define _CCCL_IF_CONSTEVAL_DEFAULT if constexpr (true)
# define _CCCL_IF_NOT_CONSTEVAL if constexpr (true)
# define _CCCL_IF_NOT_CONSTEVAL_DEFAULT if constexpr (false)
#endif // ^^^ no is constant evaluated ^^^
#if _CCCL_STD_VER >= 2020 && __cpp_char8_t >= 201811L
# define _CCCL_HAS_CHAR8_T() 1
#else // ^^^ has char8_t ^^^ / vvv no char8_t vvv
# define _CCCL_HAS_CHAR8_T() 0
#endif // ^^^ no char8_t ^^^
// We currently do not support any of the STL wchar facilities
#define _CCCL_HAS_WCHAR_T() 0
// Fixme: replace the condition with (!_CCCL_DEVICE_COMPILATION())
// FIXME: Enable this for clang-cuda in a followup
#if !_CCCL_CUDA_COMPILATION() && !defined(CCCL_DISABLE_LONG_DOUBLE_SUPPORT)
# define _CCCL_HAS_LONG_DOUBLE() 1
#else // ^^^ has long double ^^^ / vvv no long double vvv
# define _CCCL_HAS_LONG_DOUBLE() 0
#endif // ^^^ no long double ^^^
// clang-21+ and gcc-16+ allow structured bindings to introduce a pack since C++17.
#if __cpp_structured_bindings >= 202411L || _CCCL_COMPILER(CLANG, >=, 21) || _CCCL_COMPILER(GCC, >=, 16)
# define _CCCL_HAS_STRUCTURED_BINDINGS_PACK() 1
#else // ^^^ has structured bindings with pack ^^^ / vvv no structured bindings with pack vvv
# define _CCCL_HAS_STRUCTURED_BINDINGS_PACK() 0
#endif // ^^^ no structured bindings with pack ^^^
// nvcc doesn't implement structured bindings pack yet.
#if _CCCL_CUDA_COMPILER(NVCC)
# undef _CCCL_HAS_STRUCTURED_BINDINGS_PACK
# define _CCCL_HAS_STRUCTURED_BINDINGS_PACK() 0
#endif // _CCCL_CUDA_COMPILER(NVCC)
#endif // __CCCL_DIALECT_H

390
cccl_upstream/libcudacxx/include/cuda/std/__cccl/epilogue.h Normal file
View File

@@ -0,0 +1,390 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
// !!! DO NOT EDIT THIS FILE !!! This file is generated by utils/generate_prologue_epilogue.py.
// NO include guards here (this file is included multiple times)
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/diagnostic.h>
#if !defined(_CCCL_PROLOGUE_INCLUDED)
# error "cccl internal error: <cuda/std/__cccl/prologue.h> must be included before <cuda/std/__cccl/epilogue.h>"
#endif
#undef _CCCL_PROLOGUE_INCLUDED
_CCCL_NV_DIAG_POP()
_CCCL_DIAG_POP
// __declspec modifiers
#if defined(align)
# error \
"cccl internal error: macro `align` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_align)
# pragma pop_macro("align")
# undef _CCCL_POP_MACRO_align
#endif
#if defined(allocate)
# error \
"cccl internal error: macro `allocate` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_allocate)
# pragma pop_macro("allocate")
# undef _CCCL_POP_MACRO_allocate
#endif
#if defined(allocator)
# error \
"cccl internal error: macro `allocator` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_allocator)
# pragma pop_macro("allocator")
# undef _CCCL_POP_MACRO_allocator
#endif
#if defined(appdomain)
# error \
"cccl internal error: macro `appdomain` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_appdomain)
# pragma pop_macro("appdomain")
# undef _CCCL_POP_MACRO_appdomain
#endif
#if defined(code_seg)
# error \
"cccl internal error: macro `code_seg` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_code_seg)
# pragma pop_macro("code_seg")
# undef _CCCL_POP_MACRO_code_seg
#endif
#if defined(deprecated)
# error \
"cccl internal error: macro `deprecated` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_deprecated)
# pragma pop_macro("deprecated")
# undef _CCCL_POP_MACRO_deprecated
#endif
#if defined(dllimport)
# error \
"cccl internal error: macro `dllimport` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_dllimport)
# pragma pop_macro("dllimport")
# undef _CCCL_POP_MACRO_dllimport
#endif
#if defined(dllexport)
# error \
"cccl internal error: macro `dllexport` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_dllexport)
# pragma pop_macro("dllexport")
# undef _CCCL_POP_MACRO_dllexport
#endif
#if defined(empty_bases)
# error \
"cccl internal error: macro `empty_bases` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_empty_bases)
# pragma pop_macro("empty_bases")
# undef _CCCL_POP_MACRO_empty_bases
#endif
#if defined(hybrid_patchable)
# error \
"cccl internal error: macro `hybrid_patchable` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_hybrid_patchable)
# pragma pop_macro("hybrid_patchable")
# undef _CCCL_POP_MACRO_hybrid_patchable
#endif
#if defined(jitintrinsic)
# error \
"cccl internal error: macro `jitintrinsic` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_jitintrinsic)
# pragma pop_macro("jitintrinsic")
# undef _CCCL_POP_MACRO_jitintrinsic
#endif
#if defined(lifetimebound)
# error \
"cccl internal error: macro `lifetimebound` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_lifetimebound)
# pragma pop_macro("lifetimebound")
# undef _CCCL_POP_MACRO_lifetimebound
#endif
#if defined(naked)
# error \
"cccl internal error: macro `naked` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_naked)
# pragma pop_macro("naked")
# undef _CCCL_POP_MACRO_naked
#endif
#if defined(noalias)
# error \
"cccl internal error: macro `noalias` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_noalias)
# pragma pop_macro("noalias")
# undef _CCCL_POP_MACRO_noalias
#endif
#if defined(noinline)
# error \
"cccl internal error: macro `noinline` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_noinline)
# pragma pop_macro("noinline")
# undef _CCCL_POP_MACRO_noinline
#endif
#if defined(noreturn)
# error \
"cccl internal error: macro `noreturn` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_noreturn)
# pragma pop_macro("noreturn")
# undef _CCCL_POP_MACRO_noreturn
#endif
#if defined(nothrow)
# error \
"cccl internal error: macro `nothrow` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_nothrow)
# pragma pop_macro("nothrow")
# undef _CCCL_POP_MACRO_nothrow
#endif
#if defined(novtable)
# error \
"cccl internal error: macro `novtable` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_novtable)
# pragma pop_macro("novtable")
# undef _CCCL_POP_MACRO_novtable
#endif
#if defined(no_sanitize_address)
# error \
"cccl internal error: macro `no_sanitize_address` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_no_sanitize_address)
# pragma pop_macro("no_sanitize_address")
# undef _CCCL_POP_MACRO_no_sanitize_address
#endif
#if defined(process)
# error \
"cccl internal error: macro `process` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_process)
# pragma pop_macro("process")
# undef _CCCL_POP_MACRO_process
#endif
#if defined(property)
# error \
"cccl internal error: macro `property` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_property)
# pragma pop_macro("property")
# undef _CCCL_POP_MACRO_property
#endif
#if defined(restrict)
# error \
"cccl internal error: macro `restrict` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_restrict)
# pragma pop_macro("restrict")
# undef _CCCL_POP_MACRO_restrict
#endif
#if defined(safebuffers)
# error \
"cccl internal error: macro `safebuffers` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_safebuffers)
# pragma pop_macro("safebuffers")
# undef _CCCL_POP_MACRO_safebuffers
#endif
#if defined(selectany)
# error \
"cccl internal error: macro `selectany` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_selectany)
# pragma pop_macro("selectany")
# undef _CCCL_POP_MACRO_selectany
#endif
#if defined(spectre)
# error \
"cccl internal error: macro `spectre` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_spectre)
# pragma pop_macro("spectre")
# undef _CCCL_POP_MACRO_spectre
#endif
#if defined(thread)
# error \
"cccl internal error: macro `thread` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_thread)
# pragma pop_macro("thread")
# undef _CCCL_POP_MACRO_thread
#endif
#if defined(uuid)
# error \
"cccl internal error: macro `uuid` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_uuid)
# pragma pop_macro("uuid")
# undef _CCCL_POP_MACRO_uuid
#endif
// [[msvc::attribute]] attributes
#if defined(msvc)
# error \
"cccl internal error: macro `msvc` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_msvc)
# pragma pop_macro("msvc")
# undef _CCCL_POP_MACRO_msvc
#endif
#if defined(flatten)
# error \
"cccl internal error: macro `flatten` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_flatten)
# pragma pop_macro("flatten")
# undef _CCCL_POP_MACRO_flatten
#endif
#if defined(forceinline)
# error \
"cccl internal error: macro `forceinline` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_forceinline)
# pragma pop_macro("forceinline")
# undef _CCCL_POP_MACRO_forceinline
#endif
#if defined(forceinline_calls)
# error \
"cccl internal error: macro `forceinline_calls` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_forceinline_calls)
# pragma pop_macro("forceinline_calls")
# undef _CCCL_POP_MACRO_forceinline_calls
#endif
#if defined(intrinsic)
# error \
"cccl internal error: macro `intrinsic` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_intrinsic)
# pragma pop_macro("intrinsic")
# undef _CCCL_POP_MACRO_intrinsic
#endif
#if defined(noinline)
# error \
"cccl internal error: macro `noinline` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_noinline)
# pragma pop_macro("noinline")
# undef _CCCL_POP_MACRO_noinline
#endif
#if defined(noinline_calls)
# error \
"cccl internal error: macro `noinline_calls` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_noinline_calls)
# pragma pop_macro("noinline_calls")
# undef _CCCL_POP_MACRO_noinline_calls
#endif
#if defined(no_tls_guard)
# error \
"cccl internal error: macro `no_tls_guard` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_no_tls_guard)
# pragma pop_macro("no_tls_guard")
# undef _CCCL_POP_MACRO_no_tls_guard
#endif
// Windows nasty macros
#if defined(min)
# error \
"cccl internal error: macro `min` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_min)
# pragma pop_macro("min")
# undef _CCCL_POP_MACRO_min
#endif
#if defined(max)
# error \
"cccl internal error: macro `max` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_max)
# pragma pop_macro("max")
# undef _CCCL_POP_MACRO_max
#endif
#if defined(interface)
# error \
"cccl internal error: macro `interface` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_interface)
# pragma pop_macro("interface")
# undef _CCCL_POP_MACRO_interface
#endif
// sal.h on Windows
#if defined(__valid)
# error \
"cccl internal error: macro `__valid` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO___valid)
# pragma pop_macro("__valid")
# undef _CCCL_POP_MACRO___valid
#endif
#if defined(__callback)
# error \
"cccl internal error: macro `__callback` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO___callback)
# pragma pop_macro("__callback")
# undef _CCCL_POP_MACRO___callback
#endif
// other macros
#if defined(clang)
# error \
"cccl internal error: macro `clang` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_clang)
# pragma pop_macro("clang")
# undef _CCCL_POP_MACRO_clang
#endif
// sys/sysmacros.h on linux
#if defined(major)
# error \
"cccl internal error: macro `major` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_major)
# pragma pop_macro("major")
# undef _CCCL_POP_MACRO_major
#endif
#if defined(minor)
# error \
"cccl internal error: macro `minor` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_minor)
# pragma pop_macro("minor")
# undef _CCCL_POP_MACRO_minor
#endif
#if defined(makedev)
# error \
"cccl internal error: macro `makedev` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
#elif defined(_CCCL_POP_MACRO_makedev)
# pragma pop_macro("makedev")
# undef _CCCL_POP_MACRO_makedev
#endif
// NO include guards here (this file is included multiple times)

42
cccl_upstream/libcudacxx/include/cuda/std/__cccl/exceptions.h Normal file
View File

@@ -0,0 +1,42 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_EXCEPTIONS_H
#define __CCCL_EXCEPTIONS_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/execution_space.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if defined(CCCL_DISABLE_EXCEPTIONS) // Escape hatch for users to manually disable exceptions
# define _CCCL_HAS_EXCEPTIONS() 0
#elif _CCCL_COMPILER(NVRTC) // NVRTC has no exceptions
# define _CCCL_HAS_EXCEPTIONS() 0
#elif _CCCL_COMPILER(MSVC) // MSVC needs special checks for `_HAS_EXCEPTIONS` and `_CPPUNWIND`
# define _CCCL_HAS_EXCEPTIONS() ((_HAS_EXCEPTIONS != 0) && (_CPPUNWIND != 0)) // disabled with /EH
#else // other compilers use `__EXCEPTIONS`
# define _CCCL_HAS_EXCEPTIONS() (__EXCEPTIONS) // disabled with -fno-exceptions
#endif // has exceptions
#if _CCCL_HAS_EXCEPTIONS() && __cpp_constexpr_exceptions >= 202411L
# define _CCCL_HAS_CONSTEXPR_EXCEPTIONS() 1
#else // ^^^ has constexpr exceptions ^^^ / vvv no constexpr exceptions vvv
# define _CCCL_HAS_CONSTEXPR_EXCEPTIONS() 0
#endif // ^^^ no constexpr exceptions ^^^
#endif // __CCCL_EXCEPTIONS_H

87
cccl_upstream/libcudacxx/include/cuda/std/__cccl/execution_space.h Normal file
View File

@@ -0,0 +1,87 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_EXECUTION_SPACE_H
#define __CCCL_EXECUTION_SPACE_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__cccl/cuda_capabilities.h>
#if _CCCL_CUDA_COMPILATION()
# define _CCCL_HOST __host__
# define _CCCL_DEVICE __device__
# define _CCCL_HOST_DEVICE __host__ __device__
#else // ^^^ _CCCL_CUDA_COMPILATION ^^^ / vvv !_CCCL_CUDA_COMPILATION vvv
# define _CCCL_HOST
# define _CCCL_DEVICE
# define _CCCL_HOST_DEVICE
#endif // !_CCCL_CUDA_COMPILATION
#if _CCCL_TILE_COMPILATION()
# define _CCCL_TILE __tile__
#else // ^^^ _CCCL_TILE_COMPILATION() ^^^ / vvv !_CCCL_TILE_COMPILATION() vvv
# define _CCCL_TILE
#endif // ^^^ !_CCCL_TILE_COMPILATION() ^^^
// clang-cuda before version 22 requires __host__ __device__ annotations on deduction guides
#if _CCCL_CUDA_COMPILER(CLANG, <, 22)
# define _CCCL_DEDUCTION_GUIDE_ATTRIBUTES _CCCL_HOST_DEVICE
#else // ^^^ _CCCL_CUDA_COMPILER(CLANG, <, 22) ^^^ / vvv !_CCCL_CUDA_COMPILER(CLANG, <, 22) vvv
# define _CCCL_DEDUCTION_GUIDE_ATTRIBUTES
#endif // ^^ !_CCCL_CUDA_COMPILER(CLANG, <, 22) ^^^
// Global variables of non builtin types are only device accessible if they are marked as `__device__`
#if _CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC)
# define _CCCL_GLOBAL_VARIABLE _CCCL_DEVICE
#else // ^^^ _CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC) ^^^ /
// vvv !_CCCL_DEVICE_COMPILATION() || _CCCL_CUDA_COMPILER(NVHPC) vvv
# define _CCCL_GLOBAL_VARIABLE
#endif // ^^^ !_CCCL_DEVICE_COMPILATION() || _CCCL_CUDA_COMPILER(NVHPC) ^^^
#if (_CCCL_CUDA_COMPILER(NVCC, >=, 12, 8) || _CCCL_CUDA_COMPILER(NVRTC) || _CCCL_CUDA_COMPILER(CLANG, >=, 20)) \
&& _CCCL_PTX_ARCH() >= 700
# define _CCCL_HAS_GRID_CONSTANT() 1
# define _CCCL_GRID_CONSTANT __grid_constant__
#else // ^^^ has __grid_constant__ ^^^ / vvv no __grid_constant__ vvv
# define _CCCL_HAS_GRID_CONSTANT() 0
# define _CCCL_GRID_CONSTANT
#endif // ^^^ no __grid_constant__ ^^^
#if !defined(_CCCL_EXEC_CHECK_DISABLE)
# if _CCCL_CUDA_COMPILER(NVCC)
# define _CCCL_EXEC_CHECK_DISABLE _CCCL_PRAGMA(nv_exec_check_disable)
# else
# define _CCCL_EXEC_CHECK_DISABLE
# endif // _CCCL_CUDA_COMPILER(NVCC)
#endif // !_CCCL_EXEC_CHECK_DISABLE
#if _CCCL_CUDA_COMPILER(NVHPC)
# define _CCCL_TARGET_CONSTEXPR
#else // ^^^ _CCCL_CUDA_COMPILER(NVHPC) ^^^ / vvv !_CCCL_CUDA_COMPILER(NVHPC) vvv
# define _CCCL_TARGET_CONSTEXPR constexpr
#endif // ^^^ !_CCCL_CUDA_COMPILER(NVHPC) ^^^
//! @brief List of all known PTX architectures supported by this CCCL version.
#define _CCCL_KNOWN_CUDA_ARCH_LIST 50, 52, 53, 60, 61, 62, 70, 75, 80, 86, 87, 88, 89, 90, 100, 103, 110, 120, 121
//! @brief List of all known architecture specific architectures supported by this CCCL version.
#define _CCCL_KNOWN_CUDA_ARCH_SPECIFIC_LIST 90, 100, 103, 110, 120, 121
#endif // __CCCL_EXECUTION_SPACE_H

148
cccl_upstream/libcudacxx/include/cuda/std/__cccl/extended_data_types.h Normal file
View File

@@ -0,0 +1,148 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_EXTENDED_DATA_TYPES_H
#define __CCCL_EXTENDED_DATA_TYPES_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__cccl/architecture.h>
#include <cuda/std/__cccl/cuda_capabilities.h>
#include <cuda/std/__cccl/cuda_toolkit.h>
#include <cuda/std/__cccl/diagnostic.h>
#include <cuda/std/__cccl/os.h>
#include <cuda/std/__cccl/preprocessor.h>
#define _CCCL_HAS_INT128() 0
#define _CCCL_HAS_NVFP4() 0
#define _CCCL_HAS_NVFP6() 0
#define _CCCL_HAS_NVFP8() 0
#define _CCCL_HAS_NVFP16() 0
#define _CCCL_HAS_NVBF16() 0
#define _CCCL_HAS_FLOAT128() 0
#if _CCCL_TILE_COMPILATION() // TODO(miscco): Fix access to extended floating point types
# define CCCL_DISABLE_NVFP4_SUPPORT
# define CCCL_DISABLE_NVFP6_SUPPORT
# define CCCL_DISABLE_NVFP8_SUPPORT
# define CCCL_DISABLE_INT128_SUPPORT
# define CCCL_DISABLE_FLOAT128_SUPPORT
#endif // _CCCL_TILE_COMPILATION()
#if !defined(CCCL_DISABLE_INT128_SUPPORT) && _CCCL_OS(LINUX) \
&& ((_CCCL_COMPILER(NVRTC) && defined(__CUDACC_RTC_INT128__)) || defined(__SIZEOF_INT128__))
# undef _CCCL_HAS_INT128
# define _CCCL_HAS_INT128() 1
#endif
#if __has_include(<cuda_fp16.h>) && (_CCCL_HAS_CTK() || defined(LIBCUDACXX_ENABLE_HOST_NVFP16)) \
&& !defined(CCCL_DISABLE_FP16_SUPPORT)
# undef _CCCL_HAS_NVFP16
# define _CCCL_HAS_NVFP16() 1
struct __half;
struct __half2;
#endif
#if __has_include(<cuda_bf16.h>) && _CCCL_HAS_NVFP16() && !defined(CCCL_DISABLE_BF16_SUPPORT)
# undef _CCCL_HAS_NVBF16
# define _CCCL_HAS_NVBF16() 1
struct __nv_bfloat16;
struct __nv_bfloat162;
#endif
#if __has_include(<cuda_fp8.h>) && _CCCL_HAS_NVFP16() && _CCCL_HAS_NVBF16() && !defined(CCCL_DISABLE_NVFP8_SUPPORT)
# undef _CCCL_HAS_NVFP8
# define _CCCL_HAS_NVFP8() 1
struct __nv_fp8_e5m2;
struct __nv_fp8x2_e5m2;
struct __nv_fp8x4_e5m2;
struct __nv_fp8_e4m3;
struct __nv_fp8x2_e4m3;
struct __nv_fp8x4_e4m3;
# if _CCCL_CTK_AT_LEAST(12, 8)
struct __nv_fp8_e8m0;
struct __nv_fp8x2_e8m0;
struct __nv_fp8x4_e8m0;
# endif // _CCCL_CTK_AT_LEAST(12, 8)
#endif
#if __has_include(<cuda_fp6.h>) && _CCCL_HAS_NVFP8() && !_CCCL_CUDA_COMPILER(NVHPC) \
&& !defined(CCCL_DISABLE_NVFP6_SUPPORT)
# undef _CCCL_HAS_NVFP6
# define _CCCL_HAS_NVFP6() 1
struct __nv_fp6_e3m2;
struct __nv_fp6x2_e3m2;
struct __nv_fp6x4_e3m2;
struct __nv_fp6_e2m3;
struct __nv_fp6x2_e2m3;
struct __nv_fp6x4_e2m3;
#endif
#if __has_include(<cuda_fp4.h>) && _CCCL_HAS_NVFP6() && !defined(CCCL_DISABLE_NVFP4_SUPPORT)
# undef _CCCL_HAS_NVFP4
# define _CCCL_HAS_NVFP4() 1
struct __nv_fp4_e2m1;
struct __nv_fp4x2_e2m1;
struct __nv_fp4x4_e2m1;
#endif
#define _CCCL_HAS_NVFP4_E2M1() _CCCL_HAS_NVFP4()
#define _CCCL_HAS_NVFP6_E2M3() _CCCL_HAS_NVFP6()
#define _CCCL_HAS_NVFP6_E3M2() _CCCL_HAS_NVFP6()
#define _CCCL_HAS_NVFP8_E4M3() _CCCL_HAS_NVFP8()
#define _CCCL_HAS_NVFP8_E5M2() _CCCL_HAS_NVFP8()
#define _CCCL_HAS_NVFP8_E8M0() (_CCCL_HAS_NVFP8() && _CCCL_CTK_AT_LEAST(12, 8))
/***********************************************************************************************************************
* __float128
**********************************************************************************************************************/
#if !defined(CCCL_DISABLE_FLOAT128_SUPPORT) && _CCCL_HAS_INT128() && _CCCL_OS(LINUX) && !_CCCL_HOST_ARCH(ARM64) \
&& !_CCCL_TILE_COMPILATION()
// Detect host compiler support
# if (defined(__CUDACC_RTC_FLOAT128__) || defined(__SIZEOF_FLOAT128__) || defined(__FLOAT128__))
# if _CCCL_DEVICE_COMPILATION()
// Only NVCC and NVRTC 12.8+ on architectures at least SM100 supports __float128 on device
# if (_CCCL_CUDA_COMPILER(NVCC, >=, 12, 8) || _CCCL_CUDA_COMPILER(NVRTC, >=, 12, 8)) && _CCCL_PTX_ARCH() >= 1000
# undef _CCCL_HAS_FLOAT128
# define _CCCL_HAS_FLOAT128() 1
# endif // _CCCL_CUDA_COMPILER(NVCC) && _CCCL_PTX_ARCH() >= 1000
# else // ^^^ _CCCL_DEVICE_COMPILATION() ^^^ / vvv !_CCCL_DEVICE_COMPILATION() vvv
# undef _CCCL_HAS_FLOAT128
# define _CCCL_HAS_FLOAT128() 1
# endif // ^^^ !_CCCL_DEVICE_COMPILATION() ^^^
# endif // Host compiler support
#endif // !defined(CCCL_DISABLE_FLOAT128_SUPPORT) && _CCCL_HAS_INT128() && _CCCL_OS(LINUX) && !_CCCL_HOST_ARCH(ARM64)
// gcc does not allow to use q/Q floating point literals when __STRICT_ANSI__ is defined. They may be allowed by
// -fext-numeric-literals, but there is no way to detect it in the preprocessor. The user is required to define
// CCCL_GCC_HAS_EXTENDED_NUMERIC_LITERALS in this case. Otherwise, we disable the __float128 support.
//
// Note: since GCC 13, we could use f128/F128 literals, but for values > DBL_MAX, the compilation with nvcc fails due to
// "floating constant is out of range".
#if _CCCL_HAS_FLOAT128() && _CCCL_COMPILER(GCC) && defined(__STRICT_ANSI__) \
&& !defined(CCCL_GCC_HAS_EXTENDED_NUMERIC_LITERALS)
# undef _CCCL_HAS_FLOAT128
# define _CCCL_HAS_FLOAT128() 0
#endif // _CCCL_HAS_FLOAT128()
#endif // __CCCL_EXTENDED_DATA_TYPES_H

61
cccl_upstream/libcudacxx/include/cuda/std/__cccl/host_std_lib.h Normal file
View File

@@ -0,0 +1,61 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_HOST_STD_LIB_H
#define __CCCL_HOST_STD_LIB_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/preprocessor.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#define _CCCL_HOST_STD_LIB_LIBSTDCXX() _CCCL_VERSION_INVALID()
#define _CCCL_HOST_STD_LIB_LIBCXX() _CCCL_VERSION_INVALID()
#define _CCCL_HOST_STD_LIB_STL() _CCCL_VERSION_INVALID()
// include a minimal header
#if __has_include(<version>)
# include <version>
#elif __has_include(<ciso646>)
# include <ciso646>
#endif // ^^^ __has_include(<ciso646>) ^^^
#define _CCCL_HOST_STD_LIB_MAKE_VERSION(_MAJOR, _MINOR) ((_MAJOR) * 100 + (_MINOR))
#define _CCCL_HOST_STD_LIB(...) _CCCL_VERSION_COMPARE(_CCCL_HOST_STD_LIB_, _CCCL_HOST_STD_LIB_##__VA_ARGS__)
#if _CCCL_HOSTED()
# if defined(_MSVC_STL_VERSION)
# undef _CCCL_HOST_STD_LIB_STL
# define _CCCL_HOST_STD_LIB_STL() (_MSVC_STL_VERSION, 0)
# elif defined(__GLIBCXX__)
# undef _CCCL_HOST_STD_LIB_LIBSTDCXX
# define _CCCL_HOST_STD_LIB_LIBSTDCXX() (_GLIBCXX_RELEASE, 0)
# elif defined(_LIBCPP_VERSION)
# undef _CCCL_HOST_STD_LIB_LIBCXX
// since llvm-16, the version scheme has been changed from MMppp to MMmmpp
# if _LIBCPP_VERSION / 10000 < 2
# define _CCCL_HOST_STD_LIB_LIBCXX() (_LIBCPP_VERSION / 1000, 0)
# else
# define _CCCL_HOST_STD_LIB_LIBCXX() (_LIBCPP_VERSION / 10000, (_LIBCPP_VERSION / 100) % 100)
# endif
# endif // ^^^ _LIBCPP_VERSION ^^^
#endif // _CCCL_HOSTED()
#define _CCCL_HAS_HOST_STD_LIB() \
(_CCCL_HOST_STD_LIB(LIBSTDCXX) || _CCCL_HOST_STD_LIB(LIBCXX) || _CCCL_HOST_STD_LIB(STL))
#endif // __CCCL_HOST_STD_LIB_H

71
cccl_upstream/libcudacxx/include/cuda/std/__cccl/is_non_narrowing_convertible.h Normal file
View File

@@ -0,0 +1,71 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_IS_NON_NARROWING_CONVERTIBLE_H
#define __CCCL_IS_NON_NARROWING_CONVERTIBLE_H
#include <cuda/std/__cccl/compiler.h>
//! There is compiler bug that results in incorrect results for the below `__is_non_narrowing_convertible` check.
//! This breaks some common functionality, so this *must* be included outside of a system header. See nvbug4867473.
#if defined(_CCCL_FORCE_SYSTEM_HEADER_GCC) || defined(_CCCL_FORCE_SYSTEM_HEADER_CLANG) \
|| defined(_CCCL_FORCE_SYSTEM_HEADER_MSVC)
# error \
"This header must be included only within the <cuda/std/__cccl/system_header>. This most likely means a mix and match of different versions of CCCL."
#endif // system header detected
namespace __cccl_internal
{
#if _CCCL_CUDA_COMPILATION()
template <class _Tp>
__host__ __device__ _Tp&& __cccl_declval(int);
template <class _Tp>
__host__ __device__ _Tp __cccl_declval(long);
template <class _Tp>
__host__ __device__ decltype(__cccl_internal::__cccl_declval<_Tp>(0)) __cccl_declval() noexcept;
// This requires a type to be implicitly convertible (also non-arithmetic)
template <class _Tp>
__host__ __device__ void __cccl_accepts_implicit_conversion(_Tp) noexcept;
#else // ^^^ CUDA compilation ^^^ / vvv no CUDA compilation
template <class _Tp>
_Tp&& __cccl_declval(int);
template <class _Tp>
_Tp __cccl_declval(long);
template <class _Tp>
decltype(__cccl_internal::__cccl_declval<_Tp>(0)) __cccl_declval() noexcept;
// This requires a type to be implicitly convertible (also non-arithmetic)
template <class _Tp>
void __cccl_accepts_implicit_conversion(_Tp) noexcept;
#endif // no CUDA compilation
template <class...>
using __cccl_void_t = void;
template <class _Dest, class _Source, class = void>
struct __is_non_narrowing_convertible
{
static constexpr bool value = false;
};
// This also prohibits narrowing conversion in case of arithmetic types
template <class _Dest, class _Source>
struct __is_non_narrowing_convertible<_Dest,
_Source,
__cccl_void_t<decltype(__cccl_internal::__cccl_accepts_implicit_conversion<_Dest>(
__cccl_internal::__cccl_declval<_Source>())),
decltype(_Dest{__cccl_internal::__cccl_declval<_Source>()})>>
{
static constexpr bool value = true;
};
} // namespace __cccl_internal
#endif // __CCCL_IS_NON_NARROWING_CONVERTIBLE_H

120
cccl_upstream/libcudacxx/include/cuda/std/__cccl/os.h Normal file
View File

@@ -0,0 +1,120 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_OS_H
#define __CCCL_OS_H
// The header provides the following macros to determine the host architecture:
//
// _CCCL_OS(WINDOWS)
// _CCCL_OS(LINUX)
// _CCCL_OS(ANDROID)
// _CCCL_OS(QNX)
// Determine the host compiler and its version
#if defined(_WIN32) || defined(_WIN64) /* _WIN64 for NVRTC */
# define _CCCL_OS_WINDOWS_() 1
#else
# define _CCCL_OS_WINDOWS_() 0
#endif
#if defined(__linux__) || defined(__LP64__) /* __LP64__ for NVRTC */
# define _CCCL_OS_LINUX_() 1
#else
# define _CCCL_OS_LINUX_() 0
#endif
#if defined(__ANDROID__)
# define _CCCL_OS_ANDROID_() 1
#else
# define _CCCL_OS_ANDROID_() 0
#endif
#if defined(__QNX__) || defined(__QNXNTO__)
# define _CCCL_OS_QNX_() 1
#else
# define _CCCL_OS_QNX_() 0
#endif
#if defined(__APPLE__) || defined(__APPLE_CC__)
# define _CCCL_OS_APPLE_() 1
#else
# define _CCCL_OS_APPLE_() 0
#endif
#define _CCCL_OS(...) _CCCL_OS_##__VA_ARGS__##_()
//! @def CCCL_OS(os) /* implementation defined */
//!
//! @brief Detect the current operating system.
//!
//! @param os The name of the operating system to test.
//!
//! @note This macro is made available when including any libcu++ header. Users that wish to
//! include the smallest possible header for this macro should include `<cuda/std/version>`.
//!
//! For supported operating systems, the macro expands to an implementation-defined true value
//! if the current operating system matches, or false otherwise. These values may be used in
//! boolean expressions (preprocessor or otherwise), but no other guarantees are made.
//!
//! Available values for `os` include:
//!
//! - ``WINDOWS``: Windows, either in 32-bit or 64-bit mode.
//! - ``LINUX``: Any kind of Linux installation. Note that other unix-based operating systems will
//! also match against this.
//! - ``ANDROID``: Android operating system.
//! - ``QNX``: QNX real-time operating system.
//! - ``APPLE``: macOS (Intel or Apple Silicon).
//!
//! Passing any other value will result in an undefined expansion, which may or may not be
//! diagnosed by the compiler.
//!
//! @note Some operating systems may satisfy multiple conditions. For example macOS and Android
//! satisfy both `APPLE`/`ANDROID` and `LINUX`.
//!
//! @par Example
//! @code
//! #define MY_OTHER_MACRO 1
//!
//! // Expansion value can be used in ordinary macro conditionals
//! #if CCCL_OS(WINDOWS) && MY_OTHER_MACRO
//! // ...
//! #endif
//!
//! // Can be negated as usual
//! #if !CCCL_OS(QNX)
//! // ...
//! #endif
//!
//! #if CCCL_OS(APPLE)
//! // Will be visible only on macOS
//! #endif
//!
//! #if CCCL_OS(ANDROID)
//! // Will be visible only on Android
//! #endif
//!
//! #if CCCL_OS(LINUX) && !CCCL_OS(APPLE) && !CCCL_OS(ANDROID)
//! // Only visible on Linux
//! #endif
//! @endcode
//!
//! @return true if the specified OS is begin compiled for, false otherwise.
#ifdef _CCCL_DOXYGEN_INVOKED
# define CCCL_OS(os) /* implementation defined */
#else
# define CCCL_OS(__os__) _CCCL_OS_##__os__##_()
#endif
// Note: the public API is single-arg to constrain the API and allow for future expansion. The
// implementation is duplicated to guard against the OS targets being accidentally defined by
// the user.
#endif // __CCCL_OS_H

1366
cccl_upstream/libcudacxx/include/cuda/std/__cccl/preprocessor.h Normal file
View File

File diff suppressed because it is too large Load Diff

348
cccl_upstream/libcudacxx/include/cuda/std/__cccl/prologue.h Normal file
View File

@@ -0,0 +1,348 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
// !!! DO NOT EDIT THIS FILE !!! This file is generated by utils/generate_prologue_epilogue.py.
// NO include guards here (this file is included multiple times)
#if defined(_CCCL_PROLOGUE_INCLUDED)
# error \
"cccl internal error: <cuda/std/__cccl/epilogue.h> must be included before next <cuda/std/__cccl/prologue.h> is reincluded"
#endif
#define _CCCL_PROLOGUE_INCLUDED() 1
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/diagnostic.h>
#include <cuda/std/__cccl/dialect.h>
// __declspec modifiers
#if defined(align)
# pragma push_macro("align")
# undef align
# define _CCCL_POP_MACRO_align
#endif // defined(align)
#if defined(allocate)
# pragma push_macro("allocate")
# undef allocate
# define _CCCL_POP_MACRO_allocate
#endif // defined(allocate)
#if defined(allocator)
# pragma push_macro("allocator")
# undef allocator
# define _CCCL_POP_MACRO_allocator
#endif // defined(allocator)
#if defined(appdomain)
# pragma push_macro("appdomain")
# undef appdomain
# define _CCCL_POP_MACRO_appdomain
#endif // defined(appdomain)
#if defined(code_seg)
# pragma push_macro("code_seg")
# undef code_seg
# define _CCCL_POP_MACRO_code_seg
#endif // defined(code_seg)
#if defined(deprecated)
# pragma push_macro("deprecated")
# undef deprecated
# define _CCCL_POP_MACRO_deprecated
#endif // defined(deprecated)
#if defined(dllimport)
# pragma push_macro("dllimport")
# undef dllimport
# define _CCCL_POP_MACRO_dllimport
#endif // defined(dllimport)
#if defined(dllexport)
# pragma push_macro("dllexport")
# undef dllexport
# define _CCCL_POP_MACRO_dllexport
#endif // defined(dllexport)
#if defined(empty_bases)
# pragma push_macro("empty_bases")
# undef empty_bases
# define _CCCL_POP_MACRO_empty_bases
#endif // defined(empty_bases)
#if defined(hybrid_patchable)
# pragma push_macro("hybrid_patchable")
# undef hybrid_patchable
# define _CCCL_POP_MACRO_hybrid_patchable
#endif // defined(hybrid_patchable)
#if defined(jitintrinsic)
# pragma push_macro("jitintrinsic")
# undef jitintrinsic
# define _CCCL_POP_MACRO_jitintrinsic
#endif // defined(jitintrinsic)
#if defined(lifetimebound)
# pragma push_macro("lifetimebound")
# undef lifetimebound
# define _CCCL_POP_MACRO_lifetimebound
#endif // defined(lifetimebound)
#if defined(naked)
# pragma push_macro("naked")
# undef naked
# define _CCCL_POP_MACRO_naked
#endif // defined(naked)
#if defined(noalias)
# pragma push_macro("noalias")
# undef noalias
# define _CCCL_POP_MACRO_noalias
#endif // defined(noalias)
#if defined(noinline)
# pragma push_macro("noinline")
# undef noinline
# define _CCCL_POP_MACRO_noinline
#endif // defined(noinline)
#if defined(noreturn)
# pragma push_macro("noreturn")
# undef noreturn
# define _CCCL_POP_MACRO_noreturn
#endif // defined(noreturn)
#if defined(nothrow)
# pragma push_macro("nothrow")
# undef nothrow
# define _CCCL_POP_MACRO_nothrow
#endif // defined(nothrow)
#if defined(novtable)
# pragma push_macro("novtable")
# undef novtable
# define _CCCL_POP_MACRO_novtable
#endif // defined(novtable)
#if defined(no_sanitize_address)
# pragma push_macro("no_sanitize_address")
# undef no_sanitize_address
# define _CCCL_POP_MACRO_no_sanitize_address
#endif // defined(no_sanitize_address)
#if defined(process)
# pragma push_macro("process")
# undef process
# define _CCCL_POP_MACRO_process
#endif // defined(process)
#if defined(property)
# pragma push_macro("property")
# undef property
# define _CCCL_POP_MACRO_property
#endif // defined(property)
#if defined(restrict)
# pragma push_macro("restrict")
# undef restrict
# define _CCCL_POP_MACRO_restrict
#endif // defined(restrict)
#if defined(safebuffers)
# pragma push_macro("safebuffers")
# undef safebuffers
# define _CCCL_POP_MACRO_safebuffers
#endif // defined(safebuffers)
#if defined(selectany)
# pragma push_macro("selectany")
# undef selectany
# define _CCCL_POP_MACRO_selectany
#endif // defined(selectany)
#if defined(spectre)
# pragma push_macro("spectre")
# undef spectre
# define _CCCL_POP_MACRO_spectre
#endif // defined(spectre)
#if defined(thread)
# pragma push_macro("thread")
# undef thread
# define _CCCL_POP_MACRO_thread
#endif // defined(thread)
#if defined(uuid)
# pragma push_macro("uuid")
# undef uuid
# define _CCCL_POP_MACRO_uuid
#endif // defined(uuid)
// [[msvc::attribute]] attributes
#if defined(msvc)
# pragma push_macro("msvc")
# undef msvc
# define _CCCL_POP_MACRO_msvc
#endif // defined(msvc)
#if defined(flatten)
# pragma push_macro("flatten")
# undef flatten
# define _CCCL_POP_MACRO_flatten
#endif // defined(flatten)
#if defined(forceinline)
# pragma push_macro("forceinline")
# undef forceinline
# define _CCCL_POP_MACRO_forceinline
#endif // defined(forceinline)
#if defined(forceinline_calls)
# pragma push_macro("forceinline_calls")
# undef forceinline_calls
# define _CCCL_POP_MACRO_forceinline_calls
#endif // defined(forceinline_calls)
#if defined(intrinsic)
# pragma push_macro("intrinsic")
# undef intrinsic
# define _CCCL_POP_MACRO_intrinsic
#endif // defined(intrinsic)
#if defined(noinline)
# pragma push_macro("noinline")
# undef noinline
# define _CCCL_POP_MACRO_noinline
#endif // defined(noinline)
#if defined(noinline_calls)
# pragma push_macro("noinline_calls")
# undef noinline_calls
# define _CCCL_POP_MACRO_noinline_calls
#endif // defined(noinline_calls)
#if defined(no_tls_guard)
# pragma push_macro("no_tls_guard")
# undef no_tls_guard
# define _CCCL_POP_MACRO_no_tls_guard
#endif // defined(no_tls_guard)
// Windows nasty macros
#if defined(min)
# pragma push_macro("min")
# undef min
# define _CCCL_POP_MACRO_min
#endif // defined(min)
#if defined(max)
# pragma push_macro("max")
# undef max
# define _CCCL_POP_MACRO_max
#endif // defined(max)
#if defined(interface)
# pragma push_macro("interface")
# undef interface
# define _CCCL_POP_MACRO_interface
#endif // defined(interface)
// sal.h on Windows
#if defined(__valid)
# pragma push_macro("__valid")
# undef __valid
# define _CCCL_POP_MACRO___valid
#endif // defined(__valid)
#if defined(__callback)
# pragma push_macro("__callback")
# undef __callback
# define _CCCL_POP_MACRO___callback
#endif // defined(__callback)
// other macros
#if defined(clang)
# pragma push_macro("clang")
# undef clang
# define _CCCL_POP_MACRO_clang
#endif // defined(clang)
// sys/sysmacros.h on linux
#if defined(major)
# pragma push_macro("major")
# undef major
# define _CCCL_POP_MACRO_major
#endif // defined(major)
#if defined(minor)
# pragma push_macro("minor")
# undef minor
# define _CCCL_POP_MACRO_minor
#endif // defined(minor)
#if defined(makedev)
# pragma push_macro("makedev")
# undef makedev
# define _CCCL_POP_MACRO_makedev
#endif // defined(makedev)
_CCCL_DIAG_PUSH
_CCCL_NV_DIAG_PUSH()
// disable some msvc warnings
// https://github.com/microsoft/STL/blob/master/stl/inc/yvals_core.h#L353
// warning C4100: 'quack': unreferenced formal parameter
// warning C4127: conditional expression is constant
// warning C4180: qualifier applied to function type has no meaning; ignored
// warning C4197: 'purr': top-level volatile in cast is ignored
// warning C4324: 'roar': structure was padded due to alignment specifier
// warning C4455: literal suffix identifiers that do not start with an underscore are reserved
// warning C4503: 'hum': decorated name length exceeded, name was truncated
// warning C4522: 'woof' : multiple assignment operators specified
// warning C4668: 'meow' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif'
// warning C4800: 'boo': forcing value to bool 'true' or 'false' (performance warning)
// warning C4996: 'meow': was declared deprecated
_CCCL_DIAG_SUPPRESS_MSVC(4100 4127 4180 4197 4296 4324 4455 4503 4522 4668 4800 4996)
// Suppress compiler warnings about C++ extensions.
#if _CCCL_COMPILER(GCC, >=, 12)
_CCCL_DIAG_SUPPRESS_GCC("-Wc++20-extensions")
_CCCL_DIAG_SUPPRESS_GCC("-Wc++23-extensions")
#endif // _CCCL_COMPILER(GCC, >=, 12)
#if _CCCL_COMPILER(GCC, >=, 14)
_CCCL_DIAG_SUPPRESS_GCC("-Wc++26-extensions")
#endif // _CCCL_COMPILER(GCC, >=, 14)
_CCCL_DIAG_SUPPRESS_CLANG("-Wc++20-extensions")
#if _CCCL_COMPILER(CLANG, >=, 17)
_CCCL_DIAG_SUPPRESS_CLANG("-Wc++23-extensions")
_CCCL_DIAG_SUPPRESS_CLANG("-Wc++26-extensions")
#else // ^^^ _CCCL_COMPILER(CLANG, >=, 17) ^^^ / vvv _CCCL_COMPILER(CLANG, <, 17) vvv
_CCCL_DIAG_SUPPRESS_CLANG("-Wc++2b-extensions")
#endif // ^^^ _CCCL_COMPILER(CLANG, <, 17) ^^^
// Suppress `if consteval`-related warnings.
_CCCL_DIAG_SUPPRESS_NVHPC(if_consteval_nonstandard)
_CCCL_DIAG_SUPPRESS_NVHPC(is_constant_evaluated_in_nonconstexpr_context)
_CCCL_DIAG_SUPPRESS_NVHPC(if_consteval_in_nonconstexpr_function)
_CCCL_DIAG_SUPPRESS_NVCC(3215) // "if consteval" and "if not consteval" are not standard in this mode
_CCCL_DIAG_SUPPRESS_NVCC(3206) // "if consteval" and "if not consteval" are meaningless in a non-constexpr function
_CCCL_DIAG_SUPPRESS_NVCC(3060) // call to __builtin_is_constant_evaluated appearing in a non-constexpr function always
// produces "false"
// NO include guards here (this file is included multiple times)

369
cccl_upstream/libcudacxx/include/cuda/std/__cccl/ptx_isa.h Normal file
View File

@@ -0,0 +1,369 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_PTX_ISA_H_
#define __CCCL_PTX_ISA_H_
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
/*
* Targeting macros
*
* Information from:
* https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#release-notes
*/
// The first define is for future major versions of CUDACC.
// We make sure that these get the highest known PTX ISA version.
// For clang cuda check
// https://github.com/llvm/llvm-project/blob/release/<VER>.x/clang/lib/Driver/ToolChains/Cuda.cpp getNVPTXTargetFeatures
#if _CCCL_CUDACC_AT_LEAST(14, 0) && !_CCCL_CUDA_COMPILER(CLANG)
# define __cccl_ptx_isa 940ULL
// PTX ISA 9.4 is available from CUDA 13.4
#elif _CCCL_CUDACC_AT_LEAST(13, 4) && !_CCCL_CUDA_COMPILER(CLANG)
# define __cccl_ptx_isa 940ULL
// PTX ISA 9.3 is available from CUDA 13.3
#elif _CCCL_CUDACC_AT_LEAST(13, 3) && !_CCCL_CUDA_COMPILER(CLANG)
# define __cccl_ptx_isa 930ULL
// PTX ISA 9.2 is available from CUDA 13.2
#elif _CCCL_CUDACC_AT_LEAST(13, 2) && !_CCCL_CUDA_COMPILER(CLANG)
# define __cccl_ptx_isa 920ULL
// PTX ISA 9.1 is available from CUDA 13.1
#elif _CCCL_CUDACC_AT_LEAST(13, 1) && !_CCCL_CUDA_COMPILER(CLANG)
# define __cccl_ptx_isa 910ULL
// PTX ISA 9.0 is available from CUDA 13.0, driver r580
#elif _CCCL_CUDACC_AT_LEAST(13, 0) && !_CCCL_CUDA_COMPILER(CLANG)
# define __cccl_ptx_isa 900ULL
// PTX ISA 8.8 is available from CUDA 12.9, driver r575
#elif _CCCL_CUDACC_AT_LEAST(12, 9) && !_CCCL_CUDA_COMPILER(CLANG, <, 22)
# define __cccl_ptx_isa 880ULL
// PTX ISA 8.7 is available from CUDA 12.8, driver r570
#elif _CCCL_CUDACC_AT_LEAST(12, 8) && !_CCCL_CUDA_COMPILER(CLANG, <, 20)
# define __cccl_ptx_isa 870ULL
// PTX ISA 8.5 is available from CUDA 12.5, driver r555
#elif _CCCL_CUDACC_AT_LEAST(12, 5) && !_CCCL_CUDA_COMPILER(CLANG, <, 19)
# define __cccl_ptx_isa 850ULL
// PTX ISA 8.4 is available from CUDA 12.4, driver r550
#elif _CCCL_CUDACC_AT_LEAST(12, 4) && !_CCCL_CUDA_COMPILER(CLANG, <, 19)
# define __cccl_ptx_isa 840ULL
// PTX ISA 8.3 is available from CUDA 12.3, driver r545
#elif _CCCL_CUDACC_AT_LEAST(12, 3) && !_CCCL_CUDA_COMPILER(CLANG, <, 18)
# define __cccl_ptx_isa 830ULL
// PTX ISA 8.2 is available from CUDA 12.2, driver r535
#elif _CCCL_CUDACC_AT_LEAST(12, 2) && !_CCCL_CUDA_COMPILER(CLANG, <, 18)
# define __cccl_ptx_isa 820ULL
// PTX ISA 8.1 is available from CUDA 12.1, driver r530
#elif _CCCL_CUDACC_AT_LEAST(12, 1) && !_CCCL_CUDA_COMPILER(CLANG, <, 17)
# define __cccl_ptx_isa 810ULL
// PTX ISA 8.0 is available from CUDA 12.0, driver r525
#elif _CCCL_CUDACC_AT_LEAST(12, 0) && !_CCCL_CUDA_COMPILER(CLANG, <, 17)
# define __cccl_ptx_isa 800ULL
// PTX ISA 7.8 is available from CUDA 11.8, driver r520
#elif _CCCL_CUDACC_AT_LEAST(11, 8) && !_CCCL_CUDA_COMPILER(CLANG, <, 16)
# define __cccl_ptx_isa 780ULL
// PTX ISA 7.7 is available from CUDA 11.7, driver r515
#elif _CCCL_CUDACC_AT_LEAST(11, 7) && !_CCCL_CUDA_COMPILER(CLANG, <, 16)
# define __cccl_ptx_isa 770ULL
// PTX ISA 7.6 is available from CUDA 11.6, driver r510
#elif _CCCL_CUDACC_AT_LEAST(11, 6) && !_CCCL_CUDA_COMPILER(CLANG, <, 16)
# define __cccl_ptx_isa 760ULL
// PTX ISA 7.5 is available from CUDA 11.5, driver r495
#elif _CCCL_CUDACC_AT_LEAST(11, 5) && !_CCCL_CUDA_COMPILER(CLANG, <, 14)
# define __cccl_ptx_isa 750ULL
// PTX ISA 7.4 is available from CUDA 11.4, driver r470
#elif _CCCL_CUDACC_AT_LEAST(11, 4) && !_CCCL_CUDA_COMPILER(CLANG, <, 14)
# define __cccl_ptx_isa 740ULL
// PTX ISA 7.3 is available from CUDA 11.3, driver r465
#elif _CCCL_CUDACC_AT_LEAST(11, 3) && !_CCCL_CUDA_COMPILER(CLANG, <, 14)
# define __cccl_ptx_isa 730ULL
// PTX ISA 7.2 is available from CUDA 11.2, driver r460
#elif _CCCL_CUDACC_AT_LEAST(11, 2) && !_CCCL_CUDA_COMPILER(CLANG, <, 13)
# define __cccl_ptx_isa 720ULL
// PTX ISA 7.1 is available from CUDA 11.1, driver r455
#elif _CCCL_CUDACC_AT_LEAST(11, 1) && !_CCCL_CUDA_COMPILER(CLANG, <, 13)
# define __cccl_ptx_isa 710ULL
// PTX ISA 7.0 is available from CUDA 11.0, driver r445
#elif _CCCL_CUDACC_AT_LEAST(11, 0) && !_CCCL_CUDA_COMPILER(CLANG, <, 12)
# define __cccl_ptx_isa 700ULL
// Fallback case. Define the ISA version to be zero. This ensures that the macro is always defined.
#else
# define __cccl_ptx_isa 0ULL
#endif
// We define certain feature test macros depending on availability. When
// __CUDA_MINIMUM_ARCH__ is not available, we define the following features
// depending on PTX ISA. This permits checking for the feature in host code.
// When __CUDA_MINIMUM_ARCH__ is available, we only enable the feature when the
// hardware supports it.
#if __cccl_ptx_isa >= 800
# if (!defined(__CUDA_MINIMUM_ARCH__)) || (defined(__CUDA_MINIMUM_ARCH__) && 900 <= __CUDA_MINIMUM_ARCH__)
# define __cccl_lib_local_barrier_arrive_tx
# define __cccl_lib_experimental_ctk12_cp_async_exposure
# endif
#endif // __cccl_ptx_isa >= 800
// NVRTC ships a built-in copy of <nv/detail/__target_macros>, so including CCCL's version of this header will omit the
// content since the header guards are already defined. To make older NVRTC versions have a few newer feature macros
// required for the PTX tests, we define them here outside the header guards.
// TODO(bgruber): limit this workaround to NVRTC versions older than the first one shipping those macros
#if _CCCL_COMPILER(NVRTC)
// missing SM_88
# if !defined(NV_PROVIDES_SM_88)
# define _NV_TARGET_VAL_SM_88 880
# define NV_PROVIDES_SM_88 __NV_PROVIDES_SM_88
# define NV_IS_EXACTLY_SM_88 __NV_IS_EXACTLY_SM_88
# if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_88)
# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_88 1
# define _NV_TARGET___NV_IS_EXACTLY_SM_88 1
# else
# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_88 0
# define _NV_TARGET___NV_IS_EXACTLY_SM_88 0
# endif
# if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_88)
# define _NV_TARGET___NV_PROVIDES_SM_88 1
# define _NV_TARGET_BOOL___NV_PROVIDES_SM_88 1
# else
# define _NV_TARGET___NV_PROVIDES_SM_88 0
# define _NV_TARGET_BOOL___NV_PROVIDES_SM_88 0
# endif
# endif // !NV_PROVIDES_SM_88
// missing SM_90a
# ifndef NV_HAS_FEATURE_SM_90a
# define NV_HAS_FEATURE_SM_90a __NV_HAS_FEATURE_SM_90a
# if defined(__CUDA_ARCH_FEAT_SM90_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 900))
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_90a 1
# else
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_90a 0
# endif
# endif // NV_HAS_FEATURE_SM_90a
// missing SM_100
# ifndef NV_PROVIDES_SM_100
# define _NV_TARGET_VAL_SM_100 1000
# define NV_PROVIDES_SM_100 __NV_PROVIDES_SM_100
# define NV_IS_EXACTLY_SM_100 __NV_IS_EXACTLY_SM_100
# if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_100)
# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_100 1
# define _NV_TARGET___NV_IS_EXACTLY_SM_100 1
# else
# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_100 0
# define _NV_TARGET___NV_IS_EXACTLY_SM_100 0
# endif
# if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_100)
# define _NV_TARGET___NV_PROVIDES_SM_100 1
# define _NV_TARGET_BOOL___NV_PROVIDES_SM_100 1
# else
# define _NV_TARGET___NV_PROVIDES_SM_100 0
# define _NV_TARGET_BOOL___NV_PROVIDES_SM_100 0
# endif
# endif // !NV_PROVIDES_SM_100
// missing SM_100a
# ifndef NV_HAS_FEATURE_SM_100a
# define NV_HAS_FEATURE_SM_100a __NV_HAS_FEATURE_SM_100a
# if defined(__CUDA_ARCH_FEAT_SM100_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1000))
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_100a 1
# else
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_100a 0
# endif
# endif // !NV_HAS_FEATURE_SM_100a
// missing SM_103
# ifndef NV_PROVIDES_SM_103
# define _NV_TARGET_VAL_SM_103 1030
# define NV_PROVIDES_SM_103 __NV_PROVIDES_SM_103
# define NV_IS_EXACTLY_SM_103 __NV_IS_EXACTLY_SM_103
# if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_103)
# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_103 1
# define _NV_TARGET___NV_IS_EXACTLY_SM_103 1
# else
# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_103 0
# define _NV_TARGET___NV_IS_EXACTLY_SM_103 0
# endif
# if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_103)
# define _NV_TARGET___NV_PROVIDES_SM_103 1
# define _NV_TARGET_BOOL___NV_PROVIDES_SM_103 1
# else
# define _NV_TARGET___NV_PROVIDES_SM_103 0
# define _NV_TARGET_BOOL___NV_PROVIDES_SM_103 0
# endif
# endif // !NV_PROVIDES_SM_103
// missing SM_103
# ifndef NV_HAS_FEATURE_SM_103a
# define NV_HAS_FEATURE_SM_103a __NV_HAS_FEATURE_SM_103a
# if defined(__CUDA_ARCH_FEAT_SM103_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1030))
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_103a 1
# else
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_103a 0
# endif
# endif // !NV_HAS_FEATURE_SM_103a
// missing SM_110
# ifndef NV_PROVIDES_SM_110
# define _NV_TARGET_VAL_SM_110 1100
# define NV_PROVIDES_SM_110 __NV_PROVIDES_SM_110
# define NV_IS_EXACTLY_SM_110 __NV_IS_EXACTLY_SM_110
# if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_110)
# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_110 1
# define _NV_TARGET___NV_IS_EXACTLY_SM_110 1
# else
# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_110 0
# define _NV_TARGET___NV_IS_EXACTLY_SM_110 0
# endif
# if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_110)
# define _NV_TARGET___NV_PROVIDES_SM_110 1
# define _NV_TARGET_BOOL___NV_PROVIDES_SM_110 1
# else
# define _NV_TARGET___NV_PROVIDES_SM_110 0
# define _NV_TARGET_BOOL___NV_PROVIDES_SM_110 0
# endif
# endif // !NV_PROVIDES_SM_110
// missing SM_110a
# ifndef NV_HAS_FEATURE_SM_110a
# define NV_HAS_FEATURE_SM_110a __NV_HAS_FEATURE_SM_110a
# if defined(__CUDA_ARCH_FEAT_SM110_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1100))
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_110a 1
# else
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_110a 0
# endif
# endif // NV_HAS_FEATURE_SM_110a
// missing SM_120
# ifndef NV_PROVIDES_SM_120
# define _NV_TARGET_VAL_SM_120 1200
# define NV_PROVIDES_SM_120 __NV_PROVIDES_SM_120
# define NV_IS_EXACTLY_SM_120 __NV_IS_EXACTLY_SM_120
# if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_120)
# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_120 1
# define _NV_TARGET___NV_IS_EXACTLY_SM_120 1
# else
# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_120 0
# define _NV_TARGET___NV_IS_EXACTLY_SM_120 0
# endif
# if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_120)
# define _NV_TARGET___NV_PROVIDES_SM_120 1
# define _NV_TARGET_BOOL___NV_PROVIDES_SM_120 1
# else
# define _NV_TARGET___NV_PROVIDES_SM_120 0
# define _NV_TARGET_BOOL___NV_PROVIDES_SM_120 0
# endif
# endif // !NV_PROVIDES_SM_120
// missing SM_120a
# ifndef NV_HAS_FEATURE_SM_120a
# define NV_HAS_FEATURE_SM_120a __NV_HAS_FEATURE_SM_120a
# if defined(__CUDA_ARCH_FEAT_SM120_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1200))
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_120a 1
# else
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_120a 0
# endif
# endif // _CCCL_COMPILER(NVRTC)
// missing SM_121
# if !defined(NV_PROVIDES_SM_121)
# define _NV_TARGET_VAL_SM_121 1210
# define NV_PROVIDES_SM_121 __NV_PROVIDES_SM_121
# define NV_IS_EXACTLY_SM_121 __NV_IS_EXACTLY_SM_121
# if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_121)
# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_121 1
# define _NV_TARGET___NV_IS_EXACTLY_SM_121 1
# else
# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_121 0
# define _NV_TARGET___NV_IS_EXACTLY_SM_121 0
# endif
# if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_121)
# define _NV_TARGET___NV_PROVIDES_SM_121 1
# define _NV_TARGET_BOOL___NV_PROVIDES_SM_121 1
# else
# define _NV_TARGET___NV_PROVIDES_SM_121 0
# define _NV_TARGET_BOOL___NV_PROVIDES_SM_121 0
# endif
# endif // !NV_PROVIDES_SM_121
// missing SM_121a
# ifndef NV_HAS_FEATURE_SM_121a
# define NV_HAS_FEATURE_SM_121a __NV_HAS_FEATURE_SM_121a
# if defined(__CUDA_ARCH_FEAT_SM121_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1210))
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_121a 1
# else
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_121a 0
# endif
# endif // NV_HAS_FEATURE_SM_121a
//----------------------------------------------------------------------------------------------------------------------
// family-specific SM versions
// missing SM_100f
# ifndef NV_HAS_FEATURE_SM_100f
# define NV_HAS_FEATURE_SM_100f __NV_HAS_FEATURE_SM_100f
# if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1000)
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_100f 1
# else
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_100f 0
# endif
# endif // NV_HAS_FEATURE_SM_100
// missing SM_103f
# ifndef NV_HAS_FEATURE_SM_103f
# define NV_HAS_FEATURE_SM_103f __NV_HAS_FEATURE_SM_103f
# if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1030)
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_103f 1
# else
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_103f 0
# endif
# endif // NV_HAS_FEATURE_SM_103f
// missing SM_110f
# ifndef NV_HAS_FEATURE_SM_110f
# define NV_HAS_FEATURE_SM_110f __NV_HAS_FEATURE_SM_110f
# if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1100)
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_110f 1
# else
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_110f 0
# endif
# endif // NV_HAS_FEATURE_SM_110f
// missing SM_120f
# ifndef NV_HAS_FEATURE_SM_120f
# define NV_HAS_FEATURE_SM_120f __NV_HAS_FEATURE_SM_120f
# if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1200)
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_120f 1
# else
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_120f 0
# endif
# endif // NV_HAS_FEATURE_SM_120f
// missing SM_121f
# ifndef NV_HAS_FEATURE_SM_121f
# define NV_HAS_FEATURE_SM_121f __NV_HAS_FEATURE_SM_121f
# if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1210)
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_121f 1
# else
# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_121f 0
# endif
# endif // NV_HAS_FEATURE_SM_121f
#endif // _CCCL_COMPILER(NVRTC)
#endif // __CCCL_PTX_ISA_H_

72
cccl_upstream/libcudacxx/include/cuda/std/__cccl/rtti.h Normal file
View File

@@ -0,0 +1,72 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_RTTI_H
#define __CCCL_RTTI_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__cccl/builtin.h>
// NOTE: some compilers support the `typeid` feature but not the `dynamic_cast`
// feature. This is why we have separate macros for each.
#ifndef _CCCL_NO_RTTI
# if defined(CCCL_DISABLE_RTTI) // Escape hatch for users to manually disable RTTI
# define _CCCL_NO_RTTI
# elif defined(__CUDA_ARCH__)
# define _CCCL_NO_RTTI // No RTTI in CUDA device code
# elif _CCCL_COMPILER(NVRTC)
# define _CCCL_NO_RTTI
# elif _CCCL_COMPILER(MSVC)
# if _CPPRTTI == 0
# define _CCCL_NO_RTTI
# endif
# elif _CCCL_COMPILER(CLANG)
# if !_CCCL_HAS_FEATURE(cxx_rtti)
# define _CCCL_NO_RTTI
# endif
# else
# if __GXX_RTTI == 0 && __cpp_rtti == 0
# define _CCCL_NO_RTTI
# endif
# endif
#endif // !_CCCL_NO_RTTI
#ifndef _CCCL_NO_TYPEID
# if defined(CCCL_DISABLE_RTTI) // CCCL_DISABLE_RTTI disables typeid also
# define _CCCL_NO_TYPEID
# elif defined(__CUDA_ARCH__)
# define _CCCL_NO_TYPEID // No typeid in CUDA device code
# elif _CCCL_COMPILER(NVRTC)
# define _CCCL_NO_TYPEID
# elif _CCCL_COMPILER(MSVC)
// No-op, MSVC always supports typeid even when RTTI is disabled
# elif _CCCL_COMPILER(CLANG)
# if !_CCCL_HAS_FEATURE(cxx_rtti)
# define _CCCL_NO_TYPEID
# endif
# else
# if __GXX_RTTI == 0 && __cpp_rtti == 0
# define _CCCL_NO_TYPEID
# endif
# endif
#endif // !_CCCL_NO_TYPEID
#endif // __CCCL_RTTI_H

83
cccl_upstream/libcudacxx/include/cuda/std/__cccl/sequence_access.h Normal file
View File

@@ -0,0 +1,83 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_SEQUENCE_ACCESS_H
#define __CCCL_SEQUENCE_ACCESS_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
// We need to define hidden friends for {cr,r,}{begin,end} of our containers as we will otherwise encounter ambigouities
#define _CCCL_SYNTHESIZE_SEQUENCE_ACCESS(_ClassName, _ConstIter) \
[[nodiscard]] _CCCL_API friend iterator begin(_ClassName& __sequence) noexcept(noexcept(__sequence.begin())) \
{ \
return __sequence.begin(); \
} \
[[nodiscard]] _CCCL_API friend _ConstIter begin(const _ClassName& __sequence) noexcept(noexcept(__sequence.begin())) \
{ \
return __sequence.begin(); \
} \
[[nodiscard]] _CCCL_API friend iterator end(_ClassName& __sequence) noexcept(noexcept(__sequence.end())) \
{ \
return __sequence.end(); \
} \
[[nodiscard]] _CCCL_API friend _ConstIter end(const _ClassName& __sequence) noexcept(noexcept(__sequence.end())) \
{ \
return __sequence.end(); \
} \
[[nodiscard]] _CCCL_API friend _ConstIter cbegin(const _ClassName& __sequence) noexcept( \
noexcept(__sequence.begin())) \
{ \
return __sequence.begin(); \
} \
[[nodiscard]] _CCCL_API friend _ConstIter cend(const _ClassName& __sequence) noexcept(noexcept(__sequence.end())) \
{ \
return __sequence.end(); \
}
#define _CCCL_SYNTHESIZE_SEQUENCE_REVERSE_ACCESS(_ClassName, _ConstRevIter) \
[[nodiscard]] _CCCL_API friend reverse_iterator rbegin(_ClassName& __sequence) noexcept( \
noexcept(__sequence.rbegin())) \
{ \
return __sequence.rbegin(); \
} \
[[nodiscard]] _CCCL_API friend _ConstRevIter rbegin(const _ClassName& __sequence) noexcept( \
noexcept(__sequence.rbegin())) \
{ \
return __sequence.rbegin(); \
} \
[[nodiscard]] _CCCL_API friend reverse_iterator rend(_ClassName& __sequence) noexcept(noexcept(__sequence.rend())) \
{ \
return __sequence.rend(); \
} \
[[nodiscard]] _CCCL_API friend _ConstRevIter rend(const _ClassName& __sequence) noexcept( \
noexcept(__sequence.rend())) \
{ \
return __sequence.rend(); \
} \
[[nodiscard]] _CCCL_API friend _ConstRevIter crbegin(const _ClassName& __sequence) noexcept( \
noexcept(__sequence.rbegin())) \
{ \
return __sequence.rbegin(); \
} \
[[nodiscard]] _CCCL_API friend _ConstRevIter crend(const _ClassName& __sequence) noexcept( \
noexcept(__sequence.rend())) \
{ \
return __sequence.rend(); \
}
#endif // __CCCL_SEQUENCE_ACCESS_H

38
cccl_upstream/libcudacxx/include/cuda/std/__cccl/system_header.h Normal file
View File

@@ -0,0 +1,38 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_SYSTEM_HEADER_H
#define __CCCL_SYSTEM_HEADER_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/is_non_narrowing_convertible.h> // IWYU pragma: export
// Enforce that cccl headers are treated as system headers
#if _CCCL_COMPILER(GCC) || _CCCL_COMPILER(NVHPC)
# define _CCCL_FORCE_SYSTEM_HEADER_GCC
#elif _CCCL_COMPILER(CLANG)
# define _CCCL_FORCE_SYSTEM_HEADER_CLANG
#elif _CCCL_COMPILER(MSVC)
# define _CCCL_FORCE_SYSTEM_HEADER_MSVC
#endif // other compilers
// Potentially enable that cccl headers are treated as system headers
#if !defined(_CCCL_NO_SYSTEM_HEADER) && !(_CCCL_COMPILER(MSVC) && defined(_LIBCUDACXX_DISABLE_PRAGMA_MSVC_WARNING)) \
&& !_CCCL_COMPILER(NVRTC) && !defined(_LIBCUDACXX_DISABLE_PRAGMA_GCC_SYSTEM_HEADER)
# if _CCCL_COMPILER(GCC) || _CCCL_COMPILER(NVHPC)
# define _CCCL_IMPLICIT_SYSTEM_HEADER_GCC
# elif _CCCL_COMPILER(CLANG)
# define _CCCL_IMPLICIT_SYSTEM_HEADER_CLANG
# elif _CCCL_COMPILER(MSVC)
# define _CCCL_IMPLICIT_SYSTEM_HEADER_MSVC
# endif // other compilers
#endif // Use system header
#endif // __CCCL_SYSTEM_HEADER_H

31
cccl_upstream/libcudacxx/include/cuda/std/__cccl/unreachable.h Normal file
View File

@@ -0,0 +1,31 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_UNREACHABLE_H
#define __CCCL_UNREACHABLE_H
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/system_header.h>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_COMPILER(MSVC) && !_CCCL_DEVICE_COMPILATION()
# define _CCCL_UNREACHABLE() __assume(0)
#else
# define _CCCL_UNREACHABLE() __builtin_unreachable()
#endif
#endif // __CCCL_UNREACHABLE_H

26
cccl_upstream/libcudacxx/include/cuda/std/__cccl/version.h Normal file
View File

@@ -0,0 +1,26 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
// This file is somewhat automatically generated. Disable clang-format.
// clang-format off
#ifndef __CCCL_VERSION_H
#define __CCCL_VERSION_H
#define CCCL_VERSION 3005000
#define CCCL_MAJOR_VERSION (CCCL_VERSION / 1000000)
#define CCCL_MINOR_VERSION (((CCCL_VERSION / 1000) % 1000))
#define CCCL_PATCH_VERSION (CCCL_VERSION % 1000)
#if CCCL_PATCH_VERSION > 99
# error "CCCL patch version cannot be greater than 99 for compatibility with Thrust/CUB's MMMmmmpp format."
#endif
#endif // __CCCL_VERSION_H

198
cccl_upstream/libcudacxx/include/cuda/std/__cccl/visibility.h Normal file
View File

@@ -0,0 +1,198 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CCCL_VISIBILITY_H
#define __CCCL_VISIBILITY_H
#ifndef _CUDA__CCCL_CONFIG
# error "<__cccl/visibility.h> should only be included in from <cuda/__cccl_config>"
#endif // _CUDA__CCCL_CONFIG
#include <cuda/std/__cccl/compiler.h>
#include <cuda/std/__cccl/system_header.h>
// We want to ensure that all warning emitting from this header are suppressed
#if defined(_CCCL_FORCE_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_FORCE_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_FORCE_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__cccl/attributes.h>
#include <cuda/std/__cccl/cuda_capabilities.h>
#include <cuda/std/__cccl/execution_space.h>
#include <cuda/std/__cccl/os.h>
// For unknown reasons, nvc++ need to selectively disable this warning
// We do not want to use our usual macro because that would have push / pop semantics
#if _CCCL_COMPILER(NVHPC)
# pragma nv_diag_suppress 1407
#endif // _CCCL_COMPILER(NVHPC)
// Enable us to hide kernels
#if _CCCL_OS(WINDOWS) || _CCCL_COMPILER(NVRTC)
# define _CCCL_VISIBILITY_HIDDEN
#else // ^^^ _CCCL_COMPILER(NVRTC) ^^^ / vvv _CCCL_COMPILER(NVRTC) vvv
# define _CCCL_VISIBILITY_HIDDEN __attribute__((__visibility__("hidden")))
#endif // !_CCCL_COMPILER(NVRTC)
#if _CCCL_COMPILER(NVRTC)
# define _CCCL_VISIBILITY_DEFAULT
#elif _CCCL_OS(WINDOWS)
# define _CCCL_VISIBILITY_DEFAULT __declspec(dllimport)
#else // ^^^ _CCCL_COMPILER(NVRTC) ^^^ / vvv !_CCCL_COMPILER(NVRTC) vvv
# define _CCCL_VISIBILITY_DEFAULT __attribute__((__visibility__("default")))
#endif // !_CCCL_COMPILER(NVRTC)
#if _CCCL_COMPILER(NVRTC)
# define _CCCL_VISIBILITY_EXPORT
#elif _CCCL_OS(WINDOWS)
# define _CCCL_VISIBILITY_EXPORT __declspec(dllexport)
#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
# define _CCCL_VISIBILITY_EXPORT _CCCL_VISIBILITY_DEFAULT
#endif // !_CCCL_COMPILER(MSVC)
#if _CCCL_OS(WINDOWS) || _CCCL_COMPILER(NVRTC)
# define _CCCL_TYPE_VISIBILITY_DEFAULT
# define _CCCL_TYPE_VISIBILITY_HIDDEN
#elif _CCCL_HAS_ATTRIBUTE(__type_visibility__)
# define _CCCL_TYPE_VISIBILITY_DEFAULT __attribute__((__type_visibility__("default")))
# define _CCCL_TYPE_VISIBILITY_HIDDEN __attribute__((__type_visibility__("hidden")))
#else // ^^^ _CCCL_HAS_ATTRIBUTE(__type_visibility__) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(__type_visibility__) vvv
# define _CCCL_TYPE_VISIBILITY_DEFAULT _CCCL_VISIBILITY_DEFAULT
# define _CCCL_TYPE_VISIBILITY_HIDDEN _CCCL_VISIBILITY_HIDDEN
#endif // !_CCCL_COMPILER(NVRTC)
#if _CCCL_COMPILER(MSVC)
# define _CCCL_FORCEINLINE __forceinline
# define _CCCL_FORCEINLINE_LAMBDA
#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
# define _CCCL_FORCEINLINE __inline__ __attribute__((__always_inline__))
# define _CCCL_FORCEINLINE_LAMBDA __attribute__((__always_inline__))
#endif // ^^^ !_CCCL_COMPILER(MSVC) ^^^
#if _CCCL_COMPILER(NVRTC)
# define _CCCL_NOINLINE __attribute__((noinline))
#elif _CCCL_OS(WINDOWS)
# define _CCCL_NOINLINE __declspec(noinline)
#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv _CCCL_COMPILER(MSVC) vvv
// We can't use __noinline__ here because of CTK defining this macro.
# define _CCCL_NOINLINE __attribute__((noinline))
#endif // ^^^ !_CCCL_COMPILER(MSVC) ^^^
#if _CCCL_DEVICE_COMPILATION()
# define _CCCL_NOINLINE_DEVICE _CCCL_NOINLINE
#else // ^^^ _CCCL_DEVICE_COMPILATION() ^^^ / vvv !_CCCL_DEVICE_COMPILATION() vvv
# define _CCCL_NOINLINE_DEVICE
#endif // ^^^ !_CCCL_DEVICE_COMPILATION() ^^^
#if _CCCL_HAS_ATTRIBUTE(__exclude_from_explicit_instantiation__)
# define _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION __attribute__((__exclude_from_explicit_instantiation__))
#else // ^^^ exclude_from_explicit_instantiation ^^^ / vvv !exclude_from_explicit_instantiation vvv
// NVCC complains mightily about being unable to inline functions if we use _CCCL_FORCEINLINE here
# define _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
#endif // !exclude_from_explicit_instantiation
#if _CCCL_COMPILER(NVHPC) // NVHPC has issues with visibility attributes on symbols with internal linkage
# define _CCCL_HIDE_FROM_ABI inline
#else // ^^^ _CCCL_COMPILER(NVHPC) ^^^ / vvv !_CCCL_COMPILER(NVHPC) vvv
# define _CCCL_HIDE_FROM_ABI _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION inline
#endif // !_CCCL_COMPILER(NVHPC)
// Note: we will allow the user to redefine _CCCL_KERNEL_ATTRIBUTES until CCCL 4.0, since they may have
// redefined CUB_DETAIL_KERNEL_ATTRIBUTES or THRUST_DETAIL_KERNEL_ATTRIBUTES.
#if !defined(_CCCL_KERNEL_ATTRIBUTES)
# define _CCCL_KERNEL_ATTRIBUTES __global__ _CCCL_VISIBILITY_HIDDEN
#endif // !_CCCL_KERNEL_ATTRIBUTES
#if defined(CUB_DETAIL_KERNEL_ATTRIBUTES) || defined(THRUST_DETAIL_KERNEL_ATTRIBUTES)
# error \
"Redefining CCCL's kernel attributes via CUB_DETAIL_KERNEL_ATTRIBUTES or THRUST_DETAIL_KERNEL_ATTRIBUTES is not allowed. If you absolutely rely on this, you can override them by defining _CCCL_KERNEL_ATTRIBUTES, but this will be disallowed in CCCL 4.0."
#endif // !_CCCL_KERNEL_ATTRIBUTES
//! @brief \c _CCCL_HIDE_FROM_ABI and \c _CCCL_FORCEINLINE cannot be used together because
//! they both try to add `inline` to the function declaration. The following macros slice
//! the function attributes differently to avoid this problem:
//! - \c _CCCL_API declares the function host/device and hides the symbol from the ABI
//! - \c _CCCL_NODEBUG_API does the same while also hiding the function from
//! debuggers and marking the function as \c inline.
//! - \c _CCCL_TRIVIAL_API does the same as \c _CCCL_NODEBUG_API while also force-inlining
//! the function.
#if _CCCL_COMPILER(NVHPC) // NVHPC has issues with visibility attributes on symbols with internal linkage
# define _CCCL_API _CCCL_HOST_DEVICE
# define _CCCL_HOST_DEVICE_API _CCCL_HOST_DEVICE
# define _CCCL_HOST_API _CCCL_HOST
# define _CCCL_DEVICE_API _CCCL_DEVICE
# define _CCCL_TILE_API _CCCL_TILE
#else // ^^^ _CCCL_COMPILER(NVHPC) ^^^ / vvv !_CCCL_COMPILER(NVHPC) vvv
# define _CCCL_API _CCCL_TILE _CCCL_HOST_DEVICE _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
# define _CCCL_HOST_DEVICE_API _CCCL_HOST_DEVICE _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
# define _CCCL_HOST_API _CCCL_HOST _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
# define _CCCL_DEVICE_API _CCCL_DEVICE _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
# define _CCCL_TILE_API _CCCL_TILE _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
#endif // !_CCCL_COMPILER(NVHPC)
//! @brief \c _CCCL_NODEBUG_API marks a function's visibility as hidden and causes
//! debuggers to skip it. This is useful for functions like \c cuda::std::move that
//! debuggers should not step into. If a \c _CCCL_NODEBUG_API function \c F calls a normal
//! function \c G, stepping into \c F in a debugger will skip over \c F and step directly
//! into \c G. In a stacktrace, \c F will still be shone, but you will not be able to
//! set the debugger's active frame to \c F.
#define _CCCL_NODEBUG_API _CCCL_API _CCCL_ARTIFICIAL _CCCL_NODEBUG inline
#define _CCCL_NODEBUG_HOST_API _CCCL_HOST_API _CCCL_ARTIFICIAL _CCCL_NODEBUG inline
#define _CCCL_NODEBUG_DEVICE_API _CCCL_DEVICE_API _CCCL_ARTIFICIAL _CCCL_NODEBUG inline
//! @brief \c _CCCL_TRIVIAL_API force-inlines a function, marks its visibility as hidden,
//! and causes debuggers to skip it. This is useful for trivial internal functions that do
//! dispatching or other plumbing work. It is particularly useful in the definition of
//! customization point objects.
#define _CCCL_TRIVIAL_API _CCCL_API _CCCL_ARTIFICIAL _CCCL_NODEBUG _CCCL_FORCEINLINE
#define _CCCL_TRIVIAL_HOST_API _CCCL_HOST_API _CCCL_ARTIFICIAL _CCCL_NODEBUG _CCCL_FORCEINLINE
#define _CCCL_TRIVIAL_DEVICE_API _CCCL_DEVICE_API _CCCL_ARTIFICIAL _CCCL_NODEBUG _CCCL_FORCEINLINE
// Some functions have their addresses appear in public types (e.g., in
// `cuda::__overrides_for` specializations). If the function is declared
// `__attribute__((visibility("hidden")))`, and if the address appears, say, in the type
// of a member of a class that is declared `__attribute__((visibility("default")))`, GCC
// complains bitterly. So we avoid declaring those functions `hidden`. Instead of the
// typical `_CCCL_API` macro, we use `_CCCL_PUBLIC_API` for those functions.
#if _CCCL_OS(WINDOWS)
# define _CCCL_PUBLIC_API _CCCL_HOST_DEVICE
# define _CCCL_PUBLIC_HOST_API _CCCL_HOST
# define _CCCL_PUBLIC_DEVICE_API _CCCL_DEVICE
#else // ^^^ _CCCL_OS(WINDOWS) ^^^ / vvv !_CCCL_OS(WINDOWS) vvv
# define _CCCL_PUBLIC_API _CCCL_HOST_DEVICE _CCCL_VISIBILITY_DEFAULT
# define _CCCL_PUBLIC_HOST_API _CCCL_HOST _CCCL_VISIBILITY_DEFAULT
# define _CCCL_PUBLIC_DEVICE_API _CCCL_DEVICE _CCCL_VISIBILITY_DEFAULT
#endif // !_CCCL_OS(WINDOWS)
#ifdef _CCCL_DOXYGEN_INVOKED // Only for documentation
//! If defined, usage of CUDA Dynamic Parallelism is disabled and APIs launching kernels can only be called from the
//! host
# define CCCL_DISABLE_CDP
#endif // _CCCL_DOXYGEN_INVOKED
#if _CCCL_HAS_CDP()
// We have CDP, so host and device APIs can call kernels
# define _CCCL_CDP_API _CCCL_API
#else // ^^^ _CCCL_HAS_CDP() ^^^ / vvv !_CCCL_HAS_CDP() vvv
// We don't have CDP, only host APIs can call kernels
# define _CCCL_CDP_API _CCCL_HOST_API
#endif // ^^^ !_CCCL_HAS_CDP() ^^^
//! _LIBCUDACXX_HIDE_FROM_ABI is for backwards compatibility for external projects.
//! _CCCL_API and its variants are the preferred way to declare functions
//! that should be hidden from the ABI.
//! Defined here to suppress any warnings from the definition
#define _LIBCUDACXX_HIDE_FROM_ABI _CCCL_API inline
#endif // __CCCL_VISIBILITY_H