[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
EngineX CI
@@ -0,0 +1,55 @@
|
||||
//===---------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===---------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___INTERNAL_ATOMIC_H
|
||||
#define _CUDA_STD___INTERNAL_ATOMIC_H
|
||||
|
||||
#include <cuda/__cccl_config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__internal/features.h>
|
||||
|
||||
#if _CCCL_CUDA_COMPILATION()
|
||||
# define _CCCL_ATOMIC_ALWAYS_LOCK_FREE(size, ptr) (size <= 8)
|
||||
#elif _CCCL_COMPILER(CLANG) || _CCCL_COMPILER(GCC)
|
||||
# define _CCCL_ATOMIC_ALWAYS_LOCK_FREE(...) __atomic_always_lock_free(__VA_ARGS__)
|
||||
#endif // _CCCL_CUDA_COMPILER
|
||||
|
||||
// Enable bypassing automatic storage checks in atomics when using CTK 12.2 and below and if NDEBUG is defined.
|
||||
// A compiler bug prevents the safe use of `__is_local` and PTX spacep until after 13.0.
|
||||
#ifndef _CCCL_ATOMIC_UNSAFE_AUTOMATIC_STORAGE
|
||||
# if _CCCL_CUDACC_BELOW(13, 1) && !defined(NDEBUG)
|
||||
# define _CCCL_ATOMIC_UNSAFE_AUTOMATIC_STORAGE
|
||||
# endif // _CCCL_CUDACC_BELOW(13, 1)
|
||||
#endif // _CCCL_ATOMIC_UNSAFE_AUTOMATIC_STORAGE
|
||||
|
||||
#define _CCCL_ATOMIC_FLAG_TYPE int
|
||||
|
||||
// Clang provides 128b atomics as a builtin
|
||||
#if defined(CCCL_ENABLE_EXPERIMENTAL_HOST_ATOMICS_128B)
|
||||
# define _CCCL_HOST_128_ATOMICS_ENABLED() 1
|
||||
# define _CCCL_HOST_128_ATOMICS_MAYBE() 0
|
||||
// GCC does not provide 128b atomics, but they may be available as a library, this requires opt-in usage.
|
||||
// See: https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html "-mcx16" for more
|
||||
#elif _CCCL_COMPILER(CLANG) || _CCCL_COMPILER(GCC)
|
||||
# define _CCCL_HOST_128_ATOMICS_ENABLED() 0
|
||||
# define _CCCL_HOST_128_ATOMICS_MAYBE() 1
|
||||
#else
|
||||
# define _CCCL_HOST_128_ATOMICS_ENABLED() 0
|
||||
# define _CCCL_HOST_128_ATOMICS_MAYBE() 0
|
||||
#endif
|
||||
|
||||
#endif // _CUDA_STD___INTERNAL_ATOMIC_H
|
||||
@@ -0,0 +1,44 @@
|
||||
//===---------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===---------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___INTERNAL_CPP_DIALECT_H
|
||||
#define _CUDA_STD___INTERNAL_CPP_DIALECT_H
|
||||
|
||||
#include <cuda/__cccl_config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
// Define LIBCUDACXX_COMPILER_DEPRECATION macro:
|
||||
#if _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC)
|
||||
# define LIBCUDACXX_COMP_DEPR_IMPL(msg) \
|
||||
_CCCL_PRAGMA(message(__FILE__ ":" _CCCL_TO_STRING(__LINE__) ": warning: " #msg))
|
||||
#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
|
||||
# define LIBCUDACXX_COMP_DEPR_IMPL(msg) _CCCL_PRAGMA(GCC warning #msg)
|
||||
#endif // !_CCCL_COMPILER(MSVC)
|
||||
|
||||
// clang-format off
|
||||
#define LIBCUDACXX_DIALECT_DEPRECATION(REQ, CUR) \
|
||||
LIBCUDACXX_COMP_DEPR_IMPL( \
|
||||
libcu++ requires at least REQ. CUR is deprecated but still supported. CUR support will be removed in a \
|
||||
future release. Define CCCL_IGNORE_DEPRECATED_CPP_DIALECT to suppress this message.)
|
||||
// clang-format on
|
||||
|
||||
#ifndef CCCL_IGNORE_DEPRECATED_CPP_DIALECT
|
||||
# if _CCCL_STD_VER < 2017
|
||||
# error libcu++ requires at least C++ 17. Define CCCL_IGNORE_DEPRECATED_CPP_DIALECT to suppress this message.
|
||||
# endif // _CCCL_STD_VER < 2017
|
||||
#endif // CCCL_IGNORE_DEPRECATED_CPP_DIALECT
|
||||
|
||||
#endif // _CUDA_STD___INTERNAL_CPP_DIALECT_H
|
||||
127
cccl_upstream/libcudacxx/include/cuda/std/__internal/features.h
Normal file
127
cccl_upstream/libcudacxx/include/cuda/std/__internal/features.h
Normal file
@@ -0,0 +1,127 @@
|
||||
//===---------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===---------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___INTERNAL_FEATURES_H
|
||||
#define _CUDA_STD___INTERNAL_FEATURES_H
|
||||
|
||||
#include <cuda/__cccl_config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#define _LIBCUDACXX_HAS_CXX20_CHRONO_LITERALS() (!_CCCL_COMPILER(CLANG) || _CCCL_STD_VER >= 2020)
|
||||
#define _LIBCUDACXX_HAS_MONOTONIC_CLOCK() 0
|
||||
#define _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() 0
|
||||
|
||||
#if _CCCL_CUDA_COMPILATION() || __cpp_aligned_new < 201606
|
||||
# define _LIBCUDACXX_HAS_ALIGNED_ALLOCATION() 0
|
||||
#else
|
||||
# define _LIBCUDACXX_HAS_ALIGNED_ALLOCATION() 1
|
||||
#endif // !_CCCL_CUDA_COMPILATION() && __cpp_aligned_new >= 201606
|
||||
|
||||
// We need `is_constant_evaluated` for clang and gcc. MSVC also needs extensive rework
|
||||
#if !defined(_CCCL_BUILTIN_IS_CONSTANT_EVALUATED)
|
||||
# define _LIBCUDACXX_HAS_CONSTEXPR_COMPLEX_OPERATIONS() 0
|
||||
#elif _CCCL_COMPILER(NVRTC)
|
||||
# define _LIBCUDACXX_HAS_CONSTEXPR_COMPLEX_OPERATIONS() 0
|
||||
#elif _CCCL_COMPILER(MSVC)
|
||||
# define _LIBCUDACXX_HAS_CONSTEXPR_COMPLEX_OPERATIONS() 0
|
||||
#elif _CCCL_CUDA_COMPILER(CLANG)
|
||||
# define _LIBCUDACXX_HAS_CONSTEXPR_COMPLEX_OPERATIONS() 0
|
||||
#else
|
||||
# define _LIBCUDACXX_HAS_CONSTEXPR_COMPLEX_OPERATIONS() 1
|
||||
#endif
|
||||
|
||||
#if _LIBCUDACXX_HAS_CONSTEXPR_COMPLEX_OPERATIONS()
|
||||
# define _CCCL_CONSTEXPR_COMPLEX constexpr
|
||||
#else
|
||||
# define _CCCL_CONSTEXPR_COMPLEX
|
||||
#endif // !_LIBCUDACXX_HAS_CONSTEXPR_COMPLEX_OPERATIONS()
|
||||
|
||||
#ifndef _LIBCUDACXX_HAS_NO_INCOMPLETE_RANGES
|
||||
# define _LIBCUDACXX_HAS_NO_INCOMPLETE_RANGES
|
||||
#endif // _LIBCUDACXX_HAS_NO_INCOMPLETE_RANGES
|
||||
|
||||
// libcu++ requires host device support for its tests. Until then restrict usage to at least 12.2
|
||||
#if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
|
||||
# define _LIBCUDACXX_HAS_NVFP16() 1
|
||||
#else
|
||||
# define _LIBCUDACXX_HAS_NVFP16() 0
|
||||
#endif // _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
|
||||
|
||||
// libcu++ requires host device support for its tests. Until then restrict usage to at least 12.2
|
||||
#if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
|
||||
# define _LIBCUDACXX_HAS_NVBF16() 1
|
||||
#else
|
||||
# define _LIBCUDACXX_HAS_NVBF16() 0
|
||||
#endif // _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
|
||||
|
||||
#if _CCCL_COMPILER(MSVC)
|
||||
# define _CCCL_ALIGNAS_TYPE(x) alignas(x)
|
||||
# define _CCCL_ALIGNAS(x) __declspec(align(x))
|
||||
#elif _CCCL_HAS_FEATURE(cxx_alignas)
|
||||
# define _CCCL_ALIGNAS_TYPE(x) alignas(x)
|
||||
# define _CCCL_ALIGNAS(x) alignas(x)
|
||||
#else
|
||||
# define _CCCL_ALIGNAS_TYPE(x) __attribute__((__aligned__(alignof(x))))
|
||||
# define _CCCL_ALIGNAS(x) __attribute__((__aligned__(x)))
|
||||
#endif // !_CCCL_COMPILER(MSVC) && !_CCCL_HAS_FEATURE(cxx_alignas)
|
||||
|
||||
// We can only expose constexpr allocations if the compiler supports it
|
||||
// For now disable constexpr allocation support until we can actually use
|
||||
#if 0 && __cpp_constexpr_dynamic_alloc >= 201907L && __cpp_lib_constexpr_dynamic_alloc >= 201907L \
|
||||
&& _CCCL_STD_VER >= 2020 && !_CCCL_COMPILER(NVRTC)
|
||||
# define _CCCL_HAS_CONSTEXPR_ALLOCATION
|
||||
# define _CCCL_CONSTEXPR_CXX20_ALLOCATION constexpr
|
||||
#else // ^^^ has constexpr allocations ^^^ / vvv no constexpr allocations vvv
|
||||
# define _CCCL_CONSTEXPR_CXX20_ALLOCATION
|
||||
#endif // ^^^ no constexpr allocations ^^^
|
||||
|
||||
// Enable removed C++17 features
|
||||
#if defined(_LIBCUDACXX_ENABLE_CXX17_REMOVED_FEATURES)
|
||||
# define _LIBCUDACXX_ENABLE_CXX17_REMOVED_BINDERS
|
||||
#endif // _LIBCUDACXX_ENABLE_CXX17_REMOVED_FEATURES
|
||||
|
||||
#ifndef _CCCL_DISABLE_ADDITIONAL_DIAGNOSTICS
|
||||
# define _CCCL_DIAGNOSE_WARNING(_COND, _MSG) _CCCL_DIAGNOSE_IF(_COND, _MSG, "warning")
|
||||
# define _CCCL_DIAGNOSE_ERROR(_COND, _MSG) _CCCL_DIAGNOSE_IF(_COND, _MSG, "error")
|
||||
#else
|
||||
# define _CCCL_DIAGNOSE_WARNING(_COND, _MSG)
|
||||
# define _CCCL_DIAGNOSE_ERROR(_COND, _MSG)
|
||||
#endif
|
||||
|
||||
#define _CCCL_HAS_SIMD_F32X2_INTRINSICS() \
|
||||
(_CCCL_CUDACC_AT_LEAST(12, 8) && _CCCL_HAS_CTK() && !_CCCL_CUDA_COMPILER(CLANG))
|
||||
#define _CCCL_HAS_SIMD_F32X2_PTX() (__cccl_ptx_isa >= 860ULL)
|
||||
#define _CCCL_HAS_SIMD_F32X2() \
|
||||
((_CCCL_HAS_SIMD_F32X2_INTRINSICS() || _CCCL_HAS_SIMD_F32X2_PTX()) && !_CCCL_TILE_COMPILATION())
|
||||
|
||||
// nvcc >= 12.8 already optimizes 16-bit X2 min/max operations to SIMD instructions
|
||||
#define _CCCL_HAS_SIMD_16BIT_MIN_MAX_COMPILER_OPTIMIZATION() _CCCL_CUDA_COMPILER(NVCC, >=, 12, 8)
|
||||
|
||||
#define _CCCL_HAS_SIMD_8BIT_INTRINSICS() 0 // TODO(fbusato): CTK 13.2 produces non-optimal code for 8-bit SIMD instrs.
|
||||
#define _CCCL_HAS_SIMD_8BIT_PTX() (__cccl_ptx_isa >= 920ULL)
|
||||
#define _CCCL_HAS_SIMD_8BIT() \
|
||||
((_CCCL_HAS_SIMD_8BIT_PTX() || _CCCL_HAS_SIMD_8BIT_INTRINSICS()) && !_CCCL_TILE_COMPILATION())
|
||||
|
||||
// Third party libraries
|
||||
|
||||
#if (__has_include(<dlpack/dlpack.h>) || __has_include(<dlpack.h>)) && \
|
||||
!_CCCL_COMPILER(NVRTC) && !defined(CCCL_DISABLE_DLPACK)
|
||||
# define _CCCL_HAS_DLPACK() 1
|
||||
#else // ^^^ has dlpack ^^^ / vvv no dlpack vvv
|
||||
# define _CCCL_HAS_DLPACK() 0
|
||||
#endif // ^^^ no dlpack ^^^
|
||||
|
||||
#endif // _CUDA_STD___INTERNAL_FEATURES_H
|
||||
@@ -0,0 +1,188 @@
|
||||
//===---------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===---------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___INTERNAL_NAMESPACES_H
|
||||
#define _CUDA_STD___INTERNAL_NAMESPACES_H
|
||||
|
||||
#include <cuda/__cccl_config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__internal/version.h>
|
||||
|
||||
// During the header testing, we want to check if the code is wrapped by the prologue/epilogue
|
||||
#if defined(_CCCL_HEADER_TEST)
|
||||
# define _CCCL_PROLOGUE_INCLUDE_CHECK() \
|
||||
static_assert(_CCCL_PROLOGUE_INCLUDED(), "missing #include <cuda/std/__cccl/prologue.h>");
|
||||
#else // ^^^ defined(_CCCL_HEADER_TEST) ^^^ / vvv !defined(_CCCL_HEADER_TEST) vvv
|
||||
# define _CCCL_PROLOGUE_INCLUDE_CHECK()
|
||||
#endif // ^^^ !defined(_CCCL_HEADER_TEST) ^^^
|
||||
|
||||
#ifndef _LIBCUDACXX_ABI_NAMESPACE
|
||||
# define _LIBCUDACXX_ABI_NAMESPACE _CCCL_PP_CAT(__, _LIBCUDACXX_CUDA_ABI_VERSION)
|
||||
#endif // _LIBCUDACXX_ABI_NAMESPACE
|
||||
|
||||
#define _CCCL_BEGIN_NAMESPACE_NOVERSION(_NS) \
|
||||
_CCCL_PROLOGUE_INCLUDE_CHECK() namespace _NS \
|
||||
{
|
||||
#define _CCCL_END_NAMESPACE_NOVERSION(_NS) \
|
||||
} \
|
||||
_CCCL_PROLOGUE_INCLUDE_CHECK()
|
||||
#define _CCCL_BEGIN_NAMESPACE(_NS) \
|
||||
_CCCL_BEGIN_NAMESPACE_NOVERSION(_NS) inline namespace _LIBCUDACXX_ABI_NAMESPACE \
|
||||
{
|
||||
#define _CCCL_END_NAMESPACE(_NS) \
|
||||
} \
|
||||
_CCCL_END_NAMESPACE_NOVERSION(_NS)
|
||||
|
||||
// Open a namespace for APIs that were version bumped in a minor release
|
||||
// Version bump namespace should be removed from the APIs at the next major release
|
||||
#define _CCCL_BEGIN_NAMESPACE_ABI_VER4_BUMP \
|
||||
static_assert(_LIBCUDACXX_CUDA_ABI_VERSION == 4, "Version bump should be removed"); \
|
||||
inline namespace __version_bump_ver4_ \
|
||||
{
|
||||
#define _CCCL_END_NAMESPACE_ABI_VER4_BUMP \
|
||||
static_assert(_LIBCUDACXX_CUDA_ABI_VERSION == 4, "Version bump should be removed"); \
|
||||
}
|
||||
|
||||
// Standard namespaces with or without versioning
|
||||
#define _CCCL_BEGIN_NAMESPACE_CUDA_STD_NOVERSION _CCCL_BEGIN_NAMESPACE_NOVERSION(cuda::std)
|
||||
#define _CCCL_END_NAMESPACE_CUDA_STD_NOVERSION _CCCL_END_NAMESPACE_NOVERSION(cuda::std)
|
||||
#define _CCCL_BEGIN_NAMESPACE_CUDA_STD _CCCL_BEGIN_NAMESPACE(cuda::std)
|
||||
#define _CCCL_END_NAMESPACE_CUDA_STD _CCCL_END_NAMESPACE(cuda::std)
|
||||
|
||||
// cuda specific namespaces
|
||||
#define _CCCL_BEGIN_NAMESPACE_CUDA _CCCL_BEGIN_NAMESPACE(cuda)
|
||||
#define _CCCL_END_NAMESPACE_CUDA _CCCL_END_NAMESPACE(cuda)
|
||||
#define _CCCL_BEGIN_NAMESPACE_CUDA_MR _CCCL_BEGIN_NAMESPACE(cuda::mr)
|
||||
#define _CCCL_END_NAMESPACE_CUDA_MR _CCCL_END_NAMESPACE(cuda::mr)
|
||||
#define _CCCL_BEGIN_NAMESPACE_CUDA_DEVICE _CCCL_BEGIN_NAMESPACE(cuda::device)
|
||||
#define _CCCL_END_NAMESPACE_CUDA_DEVICE _CCCL_END_NAMESPACE(cuda::device)
|
||||
#define _CCCL_BEGIN_NAMESPACE_CUDA_PTX _CCCL_BEGIN_NAMESPACE(cuda::ptx)
|
||||
#define _CCCL_END_NAMESPACE_CUDA_PTX _CCCL_END_NAMESPACE(cuda::ptx)
|
||||
#define _CCCL_BEGIN_NAMESPACE_CUDA_DEVICE_EXPERIMENTAL _CCCL_BEGIN_NAMESPACE(cuda::device::experimental)
|
||||
#define _CCCL_END_NAMESPACE_CUDA_DEVICE_EXPERIMENTAL _CCCL_END_NAMESPACE(cuda::device::experimental)
|
||||
#define _CCCL_BEGIN_NAMESPACE_CUDA_DRIVER _CCCL_BEGIN_NAMESPACE(cuda::__driver)
|
||||
#define _CCCL_END_NAMESPACE_CUDA_DRIVER _CCCL_END_NAMESPACE(cuda::__driver)
|
||||
|
||||
// Namespaces related to <simd>
|
||||
#define _CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD _CCCL_BEGIN_NAMESPACE(cuda::std::simd)
|
||||
#define _CCCL_END_NAMESPACE_CUDA_STD_SIMD _CCCL_END_NAMESPACE(cuda::std::simd)
|
||||
|
||||
// Namespaces related to <ranges>
|
||||
#define _CCCL_BEGIN_NAMESPACE_CUDA_STD_RANGES _CCCL_BEGIN_NAMESPACE(cuda::std::ranges)
|
||||
#define _CCCL_END_NAMESPACE_CUDA_STD_RANGES _CCCL_END_NAMESPACE(cuda::std::ranges)
|
||||
#define _CCCL_BEGIN_NAMESPACE_CUDA_STD_VIEWS _CCCL_BEGIN_NAMESPACE(cuda::std::ranges::views)
|
||||
#define _CCCL_END_NAMESPACE_CUDA_STD_VIEWS _CCCL_END_NAMESPACE(cuda::std::ranges::views)
|
||||
|
||||
#define _CCCL_BEGIN_NAMESPACE_CPO(_CPO) \
|
||||
namespace _CPO \
|
||||
{
|
||||
#define _CCCL_END_NAMESPACE_CPO }
|
||||
|
||||
// Namespaces related to chrono / filesystem
|
||||
#define _CCCL_BEGIN_NAMESPACE_FILESYSTEM \
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD_NOVERSION \
|
||||
inline namespace __fs \
|
||||
{ \
|
||||
namespace filesystem \
|
||||
{ \
|
||||
inline namespace _LIBCUDACXX_ABI_NAMESPACE \
|
||||
{
|
||||
#define _CCCL_END_NAMESPACE_FILESYSTEM \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
_CCCL_END_NAMESPACE_CUDA_STD_NOVERSION
|
||||
|
||||
// Shorthands for different qualifiers
|
||||
// Namespaces related to execution
|
||||
#define _CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION _CCCL_BEGIN_NAMESPACE(cuda::std::execution)
|
||||
#define _CCCL_END_NAMESPACE_CUDA_STD_EXECUTION _CCCL_END_NAMESPACE(cuda::std::execution)
|
||||
|
||||
#define _CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION _CCCL_BEGIN_NAMESPACE(cuda::execution)
|
||||
#define _CCCL_END_NAMESPACE_CUDA_EXECUTION _CCCL_END_NAMESPACE(cuda::execution)
|
||||
|
||||
#define _CCCL_BEGIN_NAMESPACE_CUDA_ARGUMENT _CCCL_BEGIN_NAMESPACE(cuda::args)
|
||||
#define _CCCL_END_NAMESPACE_CUDA_ARGUMENT _CCCL_END_NAMESPACE(cuda::args)
|
||||
|
||||
// Namespace to avoid name collisions with CPOs on clang-16 (see
|
||||
// https://godbolt.org/z/9TadonrdM for example). MSVC's ancient parser also gets confused with
|
||||
// __cccl_true in the main iter_move template.
|
||||
#if _CCCL_COMPILER(CLANG, <=, 16) || _CCCL_COMPILER(MSVC)
|
||||
# define _LIBCUDACXX_BEGIN_HIDDEN_FRIEND_NAMESPACE \
|
||||
namespace __hidden \
|
||||
{
|
||||
# define _LIBCUDACXX_END_HIDDEN_FRIEND_NAMESPACE(_CLASS) \
|
||||
} \
|
||||
using __hidden::_CLASS;
|
||||
#else // ^^^ _CCCL_COMPILER(CLANG, <=, 16) ^^^ / vvv _CCCL_COMPILER(CLANG, >, 16) vvv
|
||||
# define _LIBCUDACXX_BEGIN_HIDDEN_FRIEND_NAMESPACE
|
||||
# define _LIBCUDACXX_END_HIDDEN_FRIEND_NAMESPACE(_CLASS)
|
||||
#endif // !_CCCL_COMPILER(CLANG, >, 16)
|
||||
|
||||
#if defined(CCCL_DISABLE_ARCH_DEPENDENT_NAMESPACE)
|
||||
# define _CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT
|
||||
# define _CCCL_END_NAMESPACE_ARCH_DEPENDENT
|
||||
#else // not defined(CCCL_DISABLE_ARCH_DEPENDENT_NAMESPACE)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC)
|
||||
# define _CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT \
|
||||
inline namespace _CCCL_PP_CAT(_CCCL_PP_SPLICE_WITH(_, _SM, NV_TARGET_SM_INTEGER_LIST), _NVHPC) \
|
||||
{
|
||||
# define _CCCL_END_NAMESPACE_ARCH_DEPENDENT }
|
||||
# else // ^^^ _CCCL_CUDA_COMPILER(NVHPC) ^^^ / vvv !_CCCL_CUDA_COMPILER(NVHPC) vvv
|
||||
# define _CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT \
|
||||
inline namespace _CCCL_PP_SPLICE_WITH(_, _SM, __CUDA_ARCH_LIST__) \
|
||||
{
|
||||
# define _CCCL_END_NAMESPACE_ARCH_DEPENDENT }
|
||||
# endif // ^^^ !_CCCL_CUDA_COMPILER(NVHPC) ^^^
|
||||
#endif // not defined(CCCL_DISABLE_ARCH_DEPENDENT_NAMESPACE)
|
||||
|
||||
// Host standard library namespaces
|
||||
#if _CCCL_HOST_STD_LIB(LIBSTDCXX)
|
||||
// We don't appy attributes on forward declarations, so we omit the _GLIBCXX_VISIBILITY(default)
|
||||
# if _GLIBCXX_INLINE_VERSION
|
||||
# define _CCCL_BEGIN_NAMESPACE_STD \
|
||||
_CCCL_PROLOGUE_INCLUDE_CHECK() namespace std \
|
||||
{ \
|
||||
inline _GLIBCXX_BEGIN_NAMESPACE_VERSION
|
||||
# define _CCCL_END_NAMESPACE_STD \
|
||||
_GLIBCXX_END_NAMESPACE_VERSION \
|
||||
} \
|
||||
_CCCL_PROLOGUE_INCLUDE_CHECK()
|
||||
# else // ^^^ _GLIBCXX_INLINE_VERSION ^^^ / vvv !_GLIBCXX_INLINE_VERSION vvv
|
||||
# define _CCCL_BEGIN_NAMESPACE_STD \
|
||||
_CCCL_PROLOGUE_INCLUDE_CHECK() namespace std \
|
||||
{
|
||||
# define _CCCL_END_NAMESPACE_STD \
|
||||
} \
|
||||
_CCCL_PROLOGUE_INCLUDE_CHECK()
|
||||
# endif // ^^^ !_GLIBCXX_INLINE_VERSION ^^^
|
||||
#elif _CCCL_HOST_STD_LIB(LIBCXX)
|
||||
# define _CCCL_BEGIN_NAMESPACE_STD _CCCL_PROLOGUE_INCLUDE_CHECK() _LIBCPP_BEGIN_NAMESPACE_STD
|
||||
# define _CCCL_END_NAMESPACE_STD _LIBCPP_END_NAMESPACE_STD _CCCL_PROLOGUE_INCLUDE_CHECK()
|
||||
#elif _CCCL_HOST_STD_LIB(STL)
|
||||
# define _CCCL_BEGIN_NAMESPACE_STD _CCCL_PROLOGUE_INCLUDE_CHECK() _STD_BEGIN
|
||||
# define _CCCL_END_NAMESPACE_STD _STD_END _CCCL_PROLOGUE_INCLUDE_CHECK()
|
||||
#else
|
||||
# define _CCCL_BEGIN_NAMESPACE_STD \
|
||||
_CCCL_PROLOGUE_INCLUDE_CHECK() namespace std \
|
||||
{
|
||||
# define _CCCL_END_NAMESPACE_STD \
|
||||
} \
|
||||
_CCCL_PROLOGUE_INCLUDE_CHECK()
|
||||
#endif
|
||||
|
||||
#endif // _CUDA_STD___INTERNAL_NAMESPACES_H
|
||||
@@ -0,0 +1,34 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___INTERNAL_PSTL_CONFIG_H
|
||||
#define _CUDA_STD___INTERNAL_PSTL_CONFIG_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#define _CCCL_HAS_BACKEND_CUDA() _CCCL_CUDA_COMPILATION() && !_CCCL_COMPILER(NVRTC)
|
||||
#define _CCCL_HAS_BACKEND_OMP() 0
|
||||
#define _CCCL_HAS_BACKEND_TBB() 0
|
||||
|
||||
#define _CCCL_HAS_PSTL_BACKEND() (_CCCL_HAS_BACKEND_CUDA() || _CCCL_HAS_BACKEND_OMP() || _CCCL_HAS_BACKEND_TBB())
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_STD___INTERNAL_PSTL_CONFIG_H
|
||||
@@ -0,0 +1,58 @@
|
||||
//===---------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===---------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___INTERNAL_THREAD_API_H
|
||||
#define _CUDA_STD___INTERNAL_THREAD_API_H
|
||||
|
||||
#include <cuda/__cccl_config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
// Thread API
|
||||
#ifndef _CCCL_HAS_THREAD_API_EXTERNAL
|
||||
# if _CCCL_COMPILER(NVRTC) || defined(__EMSCRIPTEN__)
|
||||
# define _CCCL_HAS_THREAD_API_EXTERNAL
|
||||
# endif
|
||||
#endif // _CCCL_HAS_THREAD_API_EXTERNAL
|
||||
|
||||
#ifndef _CCCL_HAS_THREAD_API_CUDA
|
||||
# if ((_CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC)) || defined(__EMSCRIPTEN__) || _CCCL_HOSTJIT())
|
||||
# define _CCCL_HAS_THREAD_API_CUDA
|
||||
# endif // ((_CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC)) || defined(__EMSCRIPTEN__))
|
||||
#endif // _CCCL_HAS_THREAD_API_CUDA
|
||||
|
||||
#ifndef _CCCL_HAS_THREAD_API_WIN32
|
||||
# if _CCCL_COMPILER(MSVC) && !defined(_CCCL_HAS_THREAD_API_CUDA)
|
||||
# define _CCCL_HAS_THREAD_API_WIN32
|
||||
# endif // _CCCL_COMPILER(MSVC) && !defined(_CCCL_HAS_THREAD_API_CUDA)
|
||||
#endif // _CCCL_HAS_THREAD_API_WIN32
|
||||
|
||||
#if !defined(_CCCL_HAS_THREAD_API_PTHREAD) && !defined(_CCCL_HAS_THREAD_API_WIN32) \
|
||||
&& !defined(_CCCL_HAS_THREAD_API_EXTERNAL)
|
||||
# if defined(__GNU__) || _CCCL_OS(LINUX) || _CCCL_OS(APPLE) || _CCCL_OS(QNX) \
|
||||
|| (defined(__MINGW32__) && __has_include(<pthread.h>))
|
||||
# define _CCCL_HAS_THREAD_API_PTHREAD
|
||||
# elif defined(_WIN32)
|
||||
# define _CCCL_HAS_THREAD_API_WIN32
|
||||
# else
|
||||
# define _CCCL_UNSUPPORTED_THREAD_API
|
||||
# endif // _CCCL_HAS_THREAD_API
|
||||
#endif
|
||||
|
||||
#ifndef __STDCPP_THREADS__
|
||||
# define __STDCPP_THREADS__ 1
|
||||
#endif // __STDCPP_THREADS__
|
||||
|
||||
#endif // _CUDA_STD___INTERNAL_THREAD_API_H
|
||||
@@ -0,0 +1,52 @@
|
||||
//===---------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===---------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___INTERNAL_VERSION_H
|
||||
#define _CUDA_STD___INTERNAL_VERSION_H
|
||||
|
||||
#include <cuda/__cccl_config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__cccl/version.h> // IWYU pragma: export
|
||||
|
||||
#define _LIBCUDACXX_CUDA_API_VERSION CCCL_VERSION
|
||||
#define _LIBCUDACXX_CUDA_API_VERSION_MAJOR CCCL_MAJOR_VERSION
|
||||
#define _LIBCUDACXX_CUDA_API_VERSION_MINOR CCCL_MINOR_VERSION
|
||||
#define _LIBCUDACXX_CUDA_API_VERSION_PATCH CCCL_PATCH_VERSION
|
||||
|
||||
#ifndef _LIBCUDACXX_CUDA_ABI_VERSION_LATEST
|
||||
# define _LIBCUDACXX_CUDA_ABI_VERSION_LATEST 4
|
||||
#endif
|
||||
|
||||
#ifdef _LIBCUDACXX_CUDA_ABI_VERSION
|
||||
# if _LIBCUDACXX_CUDA_ABI_VERSION != 4
|
||||
# error Unsupported libcu++ ABI version requested. Only version 4 is allowed.
|
||||
# endif
|
||||
#else
|
||||
# define _LIBCUDACXX_CUDA_ABI_VERSION _LIBCUDACXX_CUDA_ABI_VERSION_LATEST
|
||||
#endif
|
||||
|
||||
#if (_LIBCUDACXX_CUDA_ABI_VERSION < 4) && !defined(LIBCUDACXX_IGNORE_DEPRECATED_ABI)
|
||||
# error "libcu++ ABIs older than version 4 are deprecated, define LIBCUDACXX_IGNORE_DEPRECATED_ABI to ignore"
|
||||
#endif
|
||||
|
||||
#ifdef _LIBCUDACXX_PIPELINE_ASSUMED_ABI_VERSION
|
||||
# if _LIBCUDACXX_PIPELINE_ASSUMED_ABI_VERSION != _LIBCUDACXX_CUDA_ABI_VERSION
|
||||
# error cuda_pipeline.h has assumed a different libcu++ ABI version than provided by this library. To fix this, please include a libcu++ header before including cuda_pipeline.h, or upgrade to a version of the toolkit this version of libcu++ shipped in.
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#endif // _CUDA_STD___INTERNAL_VERSION_H
|
||||
Reference in New Issue
Block a user