[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
57
cccl_upstream/libcudacxx/include/cuda/std/__cstdlib/abs.h
Normal file
57
cccl_upstream/libcudacxx/include/cuda/std/__cstdlib/abs.h
Normal file
@@ -0,0 +1,57 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___CSTDLIB_ABS_H
|
||||
#define _CUDA_STD___CSTDLIB_ABS_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr int abs(int __val) noexcept
|
||||
{
|
||||
return (__val < 0) ? -__val : __val;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr long labs(long __val) noexcept
|
||||
{
|
||||
return (__val < 0l) ? -__val : __val;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr long abs(long __val) noexcept
|
||||
{
|
||||
return ::cuda::std::labs(__val);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr long long llabs(long long __val) noexcept
|
||||
{
|
||||
return (__val < 0ll) ? -__val : __val;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr long long abs(long long __val) noexcept
|
||||
{
|
||||
return ::cuda::std::llabs(__val);
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_STD___CSTDLIB_ABS_H
|
||||
@@ -0,0 +1,66 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___CSTDLIB_ALIGNED_ALLOC_H
|
||||
#define _CUDA_STD___CSTDLIB_ALIGNED_ALLOC_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__cstddef/types.h>
|
||||
#include <cuda/std/__cstdlib/malloc.h>
|
||||
#include <cuda/std/cstring>
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
# include <cstdlib>
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#include <nv/target>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if _CCCL_CUDA_COMPILATION()
|
||||
extern "C" _CCCL_DEVICE void* __cuda_syscall_aligned_malloc(size_t __nbytes, size_t __align);
|
||||
#endif // _CCCL_CUDA_COMPILATION()
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
#if !_CCCL_COMPILER(NVRTC)
|
||||
[[nodiscard]] _CCCL_HOST_API inline void*
|
||||
__aligned_alloc_host([[maybe_unused]] size_t __align, [[maybe_unused]] size_t __nbytes) noexcept
|
||||
{
|
||||
# if _CCCL_OS(WINDOWS)
|
||||
_CCCL_ASSERT(false, "Use of aligned_alloc in host code is not supported on WIndows");
|
||||
return nullptr;
|
||||
# else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
|
||||
return ::aligned_alloc(__align, __nbytes);
|
||||
# endif // ^^^ !_CCCL_COMPILER(MSVC) ^^^
|
||||
}
|
||||
#endif // !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
[[nodiscard]] _CCCL_API inline void* aligned_alloc(size_t __align, size_t __nbytes) noexcept
|
||||
{
|
||||
NV_IF_ELSE_TARGET(NV_IS_HOST,
|
||||
(return ::cuda::std::__aligned_alloc_host(__align, __nbytes);),
|
||||
(return ::__cuda_syscall_aligned_malloc(__nbytes, __align);))
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_STD___CSTDLIB_ALIGNED_ALLOC_H
|
||||
96
cccl_upstream/libcudacxx/include/cuda/std/__cstdlib/div.h
Normal file
96
cccl_upstream/libcudacxx/include/cuda/std/__cstdlib/div.h
Normal file
@@ -0,0 +1,96 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___CSTDLIB_DIV_H
|
||||
#define _CUDA_STD___CSTDLIB_DIV_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
# include <cstdlib>
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
// If available, use the host's div_t, ldiv_t, and lldiv_t types because the struct members order is
|
||||
// implementation-defined.
|
||||
#if _CCCL_HOSTED()
|
||||
using ::div_t;
|
||||
using ::ldiv_t;
|
||||
using ::lldiv_t;
|
||||
#else // ^^^ _CCCL_HOSTED() ^^^ / vvv _CCCL_FREESTANDING() vvv
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT div_t
|
||||
{
|
||||
int quot;
|
||||
int rem;
|
||||
};
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT ldiv_t
|
||||
{
|
||||
long quot;
|
||||
long rem;
|
||||
};
|
||||
|
||||
struct _CCCL_TYPE_VISIBILITY_DEFAULT lldiv_t
|
||||
{
|
||||
long long quot;
|
||||
long long rem;
|
||||
};
|
||||
#endif // _CCCL_FREESTANDING()
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr div_t div(int __x, int __y) noexcept
|
||||
{
|
||||
div_t __result{};
|
||||
__result.quot = __x / __y;
|
||||
__result.rem = __x % __y;
|
||||
return __result;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr ldiv_t ldiv(long __x, long __y) noexcept
|
||||
{
|
||||
ldiv_t __result{};
|
||||
__result.quot = __x / __y;
|
||||
__result.rem = __x % __y;
|
||||
return __result;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr ldiv_t div(long __x, long __y) noexcept
|
||||
{
|
||||
return ::cuda::std::ldiv(__x, __y);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr lldiv_t lldiv(long long __x, long long __y) noexcept
|
||||
{
|
||||
lldiv_t __result{};
|
||||
__result.quot = __x / __y;
|
||||
__result.rem = __x % __y;
|
||||
return __result;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr lldiv_t div(long long __x, long long __y) noexcept
|
||||
{
|
||||
return ::cuda::std::lldiv(__x, __y);
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_STD___CSTDLIB_DIV_H
|
||||
75
cccl_upstream/libcudacxx/include/cuda/std/__cstdlib/malloc.h
Normal file
75
cccl_upstream/libcudacxx/include/cuda/std/__cstdlib/malloc.h
Normal file
@@ -0,0 +1,75 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_STD___CSTDLIB_MALLOC_H
|
||||
#define _CUDA_STD___CSTDLIB_MALLOC_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__cmath/mul_hi.h>
|
||||
#include <cuda/std/__cstddef/types.h>
|
||||
#include <cuda/std/__cstring/memset.h>
|
||||
|
||||
#if _CCCL_HOSTED()
|
||||
# include <cstdlib>
|
||||
#endif // _CCCL_HOSTED()
|
||||
|
||||
#include <nv/target>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_STD
|
||||
|
||||
using ::free;
|
||||
using ::malloc;
|
||||
|
||||
#if _CCCL_CUDA_COMPILATION()
|
||||
[[nodiscard]] _CCCL_DEVICE_API inline void* __calloc_device(size_t __n, size_t __size) noexcept
|
||||
{
|
||||
void* __ptr{};
|
||||
|
||||
# if _CCCL_TILE_COMPILATION() // dynamic allocations are not supported in tile mode
|
||||
_CCCL_VERIFY(false, "dynamimc allocation is not supported in tile programs");
|
||||
# else // ^^^ _CCCL_TILE_COMPILATION() ^^^ / vvv !_CCCL_TILE_COMPILATION() vvv
|
||||
// check for overflow through a hypothetical larger integer
|
||||
// TODO (miscco): use `mul_overflow` once implemented
|
||||
if (::cuda::mul_hi(__n, __size) == 0)
|
||||
{
|
||||
const size_t __nbytes = __n * __size;
|
||||
__ptr = ::cuda::std::malloc(__nbytes);
|
||||
if (__ptr != nullptr)
|
||||
{
|
||||
::cuda::std::memset(__ptr, 0, __nbytes);
|
||||
}
|
||||
}
|
||||
# endif // !_CCCL_TILE_COMPILATION()
|
||||
|
||||
return __ptr;
|
||||
}
|
||||
#endif // _CCCL_CUDA_COMPILATION()
|
||||
|
||||
[[nodiscard]] _CCCL_API inline void* calloc(size_t __n, size_t __size) noexcept
|
||||
{
|
||||
NV_IF_ELSE_TARGET(NV_IS_HOST, (return ::calloc(__n, __size);), (return ::cuda::std::__calloc_device(__n, __size);))
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_STD
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_STD___CSTDLIB_MALLOC_H
|
||||
Reference in New Issue
Block a user