dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

Browse Source 
CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0
This commit is contained in:
EngineX CI
2026-07-30 09:35:51 +00:00
parent b4d01f481e
commit 56fd68e7dd
8871 changed files with 1454674 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
cccl_upstream/libcudacxx/include/cuda/__algorithm/common.h Normal file
View File

@@ -0,0 +1,68 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDA___ALGORITHM_COMMON
#define __CUDA___ALGORITHM_COMMON
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__concepts/convertible_to.h>
#include <cuda/std/__ranges/concepts.h>
#include <cuda/std/__type_traits/remove_reference.h>
#include <cuda/std/mdspan>
#include <cuda/std/span>
#include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
template <typename _Tp>
using __as_span_t = ::cuda::std::span<::cuda::std::remove_reference_t<::cuda::std::ranges::range_reference_t<_Tp>>>;
//! @brief A concept that checks if the type can be converted to a `cuda::std::span`.
//! The type must be a contiguous range.
template <typename _Tp>
_CCCL_CONCEPT __spannable = _CCCL_REQUIRES_EXPR((_Tp))( //
requires(::cuda::std::ranges::contiguous_range<_Tp>), //
requires(::cuda::std::convertible_to<_Tp, __as_span_t<_Tp>>));
template <typename _Tp>
using __as_mdspan_t =
::cuda::std::mdspan<typename ::cuda::std::decay_t<_Tp>::value_type,
typename ::cuda::std::decay_t<_Tp>::extents_type,
typename ::cuda::std::decay_t<_Tp>::layout_type,
typename ::cuda::std::decay_t<_Tp>::accessor_type>;
//! @brief A concept that checks if the type can be converted to a `cuda::std::mdspan`.
//! The type must have a conversion to `__as_mdspan_t<_Tp>`.
template <typename _Tp>
_CCCL_CONCEPT __mdspannable =
_CCCL_REQUIRES_EXPR((_Tp))(requires(::cuda::std::convertible_to<_Tp, __as_mdspan_t<_Tp>>));
template <typename _Tp>
[[nodiscard]] _CCCL_HOST_API constexpr auto __as_mdspan(_Tp&& __value) noexcept -> __as_mdspan_t<_Tp>
{
return ::cuda::std::forward<_Tp>(__value);
}
_CCCL_END_NAMESPACE_CUDA
#include <cuda/std/__cccl/epilogue.h>
#endif //__CUDA___ALGORITHM_COMMON

187
cccl_upstream/libcudacxx/include/cuda/__algorithm/copy.h Normal file
View File

@@ -0,0 +1,187 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDA___ALGORITHM_COPY_H
#define __CUDA___ALGORITHM_COPY_H
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
# include <cuda/__algorithm/common.h>
# include <cuda/__stream/launch_transform.h>
# include <cuda/__stream/stream_ref.h>
# include <cuda/__type_traits/is_trivially_copyable.h>
# include <cuda/std/__concepts/concept_macros.h>
# include <cuda/std/__exception/exception_macros.h>
# include <cuda/std/__host_stdlib/stdexcept>
# include <cuda/std/mdspan>
# include <cuda/std/span>
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
//! @brief Source access order for copy_bytes
enum class source_access_order
{
# if _CCCL_CTK_AT_LEAST(13, 0)
//! @brief Access source in stream order
stream = ::cudaMemcpySrcAccessOrderStream,
//! @brief Access source during the copy call, source can be destroyed after the API returns
during_api_call = ::cudaMemcpySrcAccessOrderDuringApiCall,
//! @brief Access source in any order, the order can change across CUDA releases
any = ::cudaMemcpySrcAccessOrderAny,
# else
any = 0x3,
# endif // _CCCL_CTK_BELOW(13, 0)
};
//! @brief Configuration for copy_bytes
struct copy_configuration
{
//! @brief Source memory location hint for copy_bytes, used only for managed memory
memory_location src_location_hint = {};
//! @brief Destination memory location hint for copy_bytes, used only for managed memory
memory_location dst_location_hint = {};
//! @brief Source access order for copy_bytes
source_access_order src_access_order = source_access_order::any;
};
namespace __detail
{
template <typename _SrcTy, typename _DstTy>
_CCCL_HOST_API void __copy_bytes_impl(
stream_ref __stream,
::cuda::std::span<_SrcTy> __src,
::cuda::std::span<_DstTy> __dst,
[[maybe_unused]] copy_configuration __config)
{
static_assert(!::cuda::std::is_const_v<_DstTy>, "Copy destination can't be const");
static_assert(::cuda::is_trivially_copyable_v<_SrcTy> && ::cuda::is_trivially_copyable_v<_DstTy>);
if (__src.size_bytes() > __dst.size_bytes())
{
_CCCL_THROW(::std::invalid_argument, "Copy destination is too small to fit the source data");
}
if (__src.size_bytes() == 0)
{
return;
}
# if _CCCL_CTK_AT_LEAST(13, 0)
CUmemcpyAttributes __attributes = {};
__attributes.srcAccessOrder = static_cast<::CUmemcpySrcAccessOrder>(__config.src_access_order);
__attributes.srcLocHint.id = __config.src_location_hint.id;
__attributes.srcLocHint.type = static_cast<::CUmemLocationType>(__config.src_location_hint.type);
__attributes.dstLocHint.id = __config.dst_location_hint.id;
__attributes.dstLocHint.type = static_cast<::CUmemLocationType>(__config.dst_location_hint.type);
::cuda::__ensure_current_context guard(__stream);
::cuda::__driver::__memcpyAsyncWithAttributes(
__dst.data(), __src.data(), __src.size_bytes(), __stream.get(), __attributes);
# else
::cuda::__driver::__memcpyAsync(__dst.data(), __src.data(), __src.size_bytes(), __stream.get());
# endif // _CCCL_CTK_BELOW(13, 0)
}
template <typename _SrcElem,
typename _SrcExtents,
typename _SrcLayout,
typename _SrcAccessor,
typename _DstElem,
typename _DstExtents,
typename _DstLayout,
typename _DstAccessor>
_CCCL_HOST_API void __copy_bytes_impl(
stream_ref __stream,
::cuda::std::mdspan<_SrcElem, _SrcExtents, _SrcLayout, _SrcAccessor> __src,
::cuda::std::mdspan<_DstElem, _DstExtents, _DstLayout, _DstAccessor> __dst,
copy_configuration __config)
{
static_assert(::cuda::std::is_constructible_v<_DstExtents, _SrcExtents>,
"Multidimensional copy requires both source and destination extents to be compatible");
static_assert(::cuda::std::is_same_v<_SrcLayout, _DstLayout>,
"Multidimensional copy requires both source and destination layouts to match");
// Check only destination, because the layout of destination is the same as source
if (!__dst.is_exhaustive())
{
_CCCL_THROW(::std::invalid_argument, "copy_bytes supports only exhaustive mdspans");
}
if (__src.extents() != __dst.extents())
{
_CCCL_THROW(::std::invalid_argument, "Copy destination size differs from the source");
}
::cuda::__detail::__copy_bytes_impl(
__stream,
::cuda::std::span(__src.data_handle(), __src.mapping().required_span_size()),
::cuda::std::span(__dst.data_handle(), __dst.mapping().required_span_size()),
__config);
}
} // namespace __detail
//! @brief Launches a bytewise memory copy from source to destination into the provided
//! stream.
//!
//! Both source and destination needs to be a `contiguous_range` and convert to
//! `cuda::std::span`. The element types of both the source and destination range is
//! required to be trivially copyable.
//!
//! This call might be synchronous if either source or destination is pagable host memory.
//! It will be synchronous if both destination and copy is located in host memory.
//!
//! @param __stream Stream that the copy should be inserted into
//! @param __src Source to copy from
//! @param __dst Destination to copy into
//! @param __config Configuration for the copy
_CCCL_TEMPLATE(typename _SrcTy, typename _DstTy)
_CCCL_REQUIRES( // NOLINT(modernize-type-traits)
__spannable<transformed_device_argument_t<_SrcTy>> _CCCL_AND __spannable<transformed_device_argument_t<_DstTy>>)
_CCCL_HOST_API void copy_bytes(stream_ref __stream, _SrcTy&& __src, _DstTy&& __dst, copy_configuration __config = {})
{
::cuda::__detail::__copy_bytes_impl(
__stream,
::cuda::std::span(launch_transform(__stream, ::cuda::std::forward<_SrcTy>(__src))),
::cuda::std::span(launch_transform(__stream, ::cuda::std::forward<_DstTy>(__dst))),
__config);
}
//! @overload
//! @note This overload accepts mdspan-compatible types.
_CCCL_TEMPLATE(typename _SrcTy, typename _DstTy)
_CCCL_REQUIRES( // NOLINT(modernize-type-traits)
__mdspannable<transformed_device_argument_t<_SrcTy>> _CCCL_AND __mdspannable<transformed_device_argument_t<_DstTy>>)
_CCCL_HOST_API void copy_bytes(stream_ref __stream, _SrcTy&& __src, _DstTy&& __dst, copy_configuration __config = {})
{
::cuda::__detail::__copy_bytes_impl(
__stream,
::cuda::__as_mdspan(launch_transform(__stream, ::cuda::std::forward<_SrcTy>(__src))),
::cuda::__as_mdspan(launch_transform(__stream, ::cuda::std::forward<_DstTy>(__dst))),
__config);
}
_CCCL_END_NAMESPACE_CUDA
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
#endif // __CUDA___ALGORITHM_COPY_H

101
cccl_upstream/libcudacxx/include/cuda/__algorithm/fill.h Normal file
View File

@@ -0,0 +1,101 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDA___ALGORITHM_FILL
#define __CUDA___ALGORITHM_FILL
#include <cuda/std/detail/__config>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
# include <cuda/__algorithm/common.h>
# include <cuda/__stream/launch_transform.h>
# include <cuda/__stream/stream_ref.h>
# include <cuda/__type_traits/is_trivially_copyable.h>
# include <cuda/std/__concepts/concept_macros.h>
# include <cuda/std/__exception/exception_macros.h>
# include <cuda/std/__host_stdlib/stdexcept>
# include <cuda/std/__cccl/prologue.h>
_CCCL_BEGIN_NAMESPACE_CUDA
namespace __detail
{
template <typename _DstTy, ::cuda::std::size_t _DstSize>
_CCCL_HOST_API void
__fill_bytes_impl(stream_ref __stream, ::cuda::std::span<_DstTy, _DstSize> __dst, ::cuda::std::uint8_t __value)
{
static_assert(!::cuda::std::is_const_v<_DstTy>, "Fill destination can't be const");
static_assert(::cuda::is_trivially_copyable_v<_DstTy>);
// TODO do a host callback if not device accessible?
::cuda::__driver::__memsetAsync(__dst.data(), __value, __dst.size_bytes(), __stream.get());
}
template <typename _DstElem, typename _DstExtents, typename _DstLayout, typename _DstAccessor>
_CCCL_HOST_API void __fill_bytes_impl(stream_ref __stream,
::cuda::std::mdspan<_DstElem, _DstExtents, _DstLayout, _DstAccessor> __dst,
::cuda::std::uint8_t __value)
{
// Check if the mdspan is exhaustive
if (!__dst.is_exhaustive())
{
_CCCL_THROW(::std::invalid_argument, "fill_bytes supports only exhaustive mdspans");
}
::cuda::__detail::__fill_bytes_impl(
__stream, ::cuda::std::span(__dst.data_handle(), __dst.mapping().required_span_size()), __value);
}
} // namespace __detail
//! @brief Launches an operation to bytewise fill the memory into the provided stream.
//!
//! The destination needs to be or launch_transform to a `contiguous_range` and convert to `cuda::std::span`.
//! The element type of the destination is required to be trivially copyable.
//!
//! The destination cannot reside in pagable host memory.
//!
//! @param __stream Stream that the copy should be inserted into
//! @param __dst Destination memory to fill
//! @param __value Value to fill into every byte in the destination
_CCCL_TEMPLATE(typename _DstTy)
_CCCL_REQUIRES(__spannable<transformed_device_argument_t<_DstTy>>)
_CCCL_HOST_API void fill_bytes(stream_ref __stream, _DstTy&& __dst, ::cuda::std::uint8_t __value)
{
::cuda::__detail::__fill_bytes_impl(
__stream, ::cuda::std::span(launch_transform(__stream, ::cuda::std::forward<_DstTy>(__dst))), __value);
}
//! @overload
//! @note This overload accepts mdspan-compatible types.
_CCCL_TEMPLATE(typename _DstTy)
_CCCL_REQUIRES(__mdspannable<transformed_device_argument_t<_DstTy>>)
_CCCL_HOST_API void fill_bytes(stream_ref __stream, _DstTy&& __dst, ::cuda::std::uint8_t __value)
{
::cuda::__detail::__fill_bytes_impl(
__stream, __as_mdspan(launch_transform(__stream, ::cuda::std::forward<_DstTy>(__dst))), __value);
}
_CCCL_END_NAMESPACE_CUDA
# include <cuda/std/__cccl/epilogue.h>
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
#endif // __CUDA___ALGORITHM_FILL