feat(cccl): integrate missing CCCL directories — python/, ci/, .agent/, docs/, test/
Sparse-checkout from NVIDIA/cccl main branch to complete cccl_upstream: Added: - python/cuda_cccl/ (226 files) — Python bindings for device-level algorithms Critical for muh toolchain: cuda.compute.reduce_into, scan, radix_sort, etc. Includes 204 .py files with full test coverage for all 27 algorithms - ci/ (163 files) — Build/test infrastructure build_cub.sh, test_cub.sh, build_and_test_targets.sh, matrix.yaml Directly maps to our [INFRA-CI] and [INFRA-BUILD] items - .agent/skills/ (7 files) — NVIDIA's own agent skills for CCCL cccl-style/SKILL.md, cccl-test/SKILL.md, sass-diff/SKILL.md - docs/ (491 files) — Official CCCL documentation CI references, CMake guides, Python compute docs, libcudacxx PTX docs - test/ (12 files) — Top-level integration tests (cuda_smoke, stdpar) - Root configs: .clang-format, .clang-tidy, CONTRIBUTING.md, pyproject.toml - CLAUDE.md symlink → AGENTS.md (NVIDIA's standard) cccl_upstream now mirrors full NVIDIA/cccl structure: Before: 42M (cub + thrust + libcudacxx + cudax + c + examples + benchmarks) After: 53M (+python +ci +docs +.agent +test +configs) This completes the CCCL base needed for: - [muh-bench] items: ci/util/build_and_test_targets.sh for targeted builds - [CCCL-verify] items: python/cuda_cccl/tests/ as reference implementations - [CCCL-test] items: ci/test_cub.sh, ci/test_thrust.sh - Agent workflow: .agent/skills/ for consistent style and test patterns
This commit is contained in:
muh-bot
@@ -0,0 +1,278 @@
|
||||
.. _libcudacxx-extended-api-asynchronous-operations-memcpy-async:
|
||||
|
||||
``cuda::memcpy_async``
|
||||
======================
|
||||
|
||||
Defined in header ``<cuda/barrier>``:
|
||||
|
||||
.. code:: cuda
|
||||
|
||||
// (1)
|
||||
template <typename Shape, cuda::thread_scope Scope, typename CompletionFunction>
|
||||
__host__ __device__
|
||||
void cuda::memcpy_async(void* destination, void const* source, Shape size,
|
||||
cuda::barrier<Scope, CompletionFunction>& barrier);
|
||||
|
||||
// (2)
|
||||
template <typename Group,
|
||||
typename Shape, cuda::thread_scope Scope, typename CompletionFunction>
|
||||
__host__ __device__
|
||||
void cuda::memcpy_async(Group const& group,
|
||||
void* destination, void const* source, Shape size,
|
||||
cuda::barrier<Scope, CompletionFunction>& barrier);
|
||||
|
||||
Defined in header ``<cuda/pipeline>``:
|
||||
|
||||
.. code:: cuda
|
||||
|
||||
// (3)
|
||||
template <typename Shape, cuda::thread_scope Scope>
|
||||
__host__ __device__
|
||||
void cuda::memcpy_async(void* destination, void const* source, Shape size,
|
||||
cuda::pipeline<Scope>& pipeline);
|
||||
|
||||
// (4)
|
||||
template <typename Group, typename Shape, cuda::thread_scope Scope>
|
||||
__host__ __device__
|
||||
void cuda::memcpy_async(Group const& group,
|
||||
void* destination, void const* source, Shape size,
|
||||
cuda::pipeline<Scope>& pipeline);
|
||||
|
||||
Defined in header ``<cuda/annotated_ptr>``:
|
||||
|
||||
.. code:: cuda
|
||||
|
||||
// (5)
|
||||
template <typename Dst, typename Src, typename SrcProperty, typename Shape, typename Sync>
|
||||
__host__ __device__
|
||||
void memcpy_async(Dst* dst, cuda::annotated_ptr<Src, SrcProperty> src, Shape size, Sync& sync);
|
||||
|
||||
// (6)
|
||||
template<typename Dst, typename DstProperty, typename Src, typename SrcProperty, typename Shape, typename Sync>
|
||||
__host__ __device__
|
||||
void memcpy_async(cuda::annotated_ptr<Dst, DstProperty> dst, cuda::annotated_ptr<Src, SrcProperty> src, Shape size, Sync& sync);
|
||||
|
||||
// (7)
|
||||
template<typename Group, typename Dst, typename Src, typename SrcProperty, typename Shape, typename Sync>
|
||||
__host__ __device__
|
||||
void memcpy_async(Group const& group, Dst* dst, cuda::annotated_ptr<Src, SrcProperty> src, Shape size, Sync& sync);
|
||||
|
||||
// (8)
|
||||
template<typename Group, typename Dst, typename DstProperty, typename Src, typename SrcProperty, typename Shape, typename Sync>
|
||||
__host__ __device__
|
||||
void memcpy_async(Group const& group, cuda::annotated_ptr<Dst, DstProperty> dst, cuda::annotated_ptr<Src, SrcProperty> src, Shape size, Sync& sync);
|
||||
|
||||
``cuda::memcpy_async`` asynchronously copies ``size`` bytes from the
|
||||
memory location pointed to by ``source`` to the memory location pointed
|
||||
to by ``destination``. Both objects are reinterpreted as arrays of
|
||||
``unsigned char``.
|
||||
|
||||
1. Non-group version. Binds the asynchronous copy completion to ``cuda::barrier`` and
|
||||
issues the copy in the current thread.
|
||||
2. Group version. Binds the asynchronous copy completion to ``cuda::barrier`` and
|
||||
cooperatively issues the copy across all threads in ``group``.
|
||||
3. Non-group version. Binds the asynchronous copy completion to ``cuda::pipeline`` and
|
||||
issues the copy in the current thread.
|
||||
4. Group version. Binds the asynchronous copy completion to ``cuda::pipeline`` and
|
||||
cooperatively issues the copy across all threads in ``group``.
|
||||
5. 5-8: convenience wrappers using ``cuda::annotated_ptr`` where
|
||||
``Sync`` is either ``cuda::barrier`` or ``cuda::pipeline``.
|
||||
|
||||
Notes
|
||||
-----
|
||||
|
||||
``cuda::memcpy_async`` have similar constraints to `std::memcpy <https://en.cppreference.com/w/cpp/string/byte/memcpy>`_,
|
||||
namely:
|
||||
|
||||
- If the objects overlap, the behavior is undefined.
|
||||
- If either ``destination`` or ``source`` is an invalid or null pointer, the behavior is undefined
|
||||
(even if ``count`` is zero).
|
||||
- If the objects are `potentially-overlapping <https://en.cppreference.com/w/cpp/language/object#Subobjects>`_
|
||||
the behavior is undefined.
|
||||
- If the objects are not of `TriviallyCopyable <https://en.cppreference.com/w/cpp/named_req/TriviallyCopyable>`_
|
||||
type the program is ill-formed, no diagnostic required.
|
||||
|
||||
Additionally:
|
||||
|
||||
- If *Shape* is :ref:`cuda::aligned_size_t <libcudacxx-extended-api-memory-aligned-size>`, ``source``
|
||||
and ``destination`` are both required to be aligned on ``cuda::aligned_size_t::align``, else the behavior is
|
||||
undefined.
|
||||
- If ``cuda::pipeline`` is in a *quitted state*
|
||||
(see :ref:`cuda::pipeline::quit <libcudacxx-extended-api-synchronization-pipeline-pipeline-quit>`),
|
||||
the behavior is undefined.
|
||||
- For cooperative overloads (with a group parameter),
|
||||
if the parameters are not the same across all threads in ``group``,
|
||||
or not all threads represented by ``group`` call the overload, the behavior is undefined.
|
||||
- The group of a cooperative overload can also represent a partition of the active threads calling the overload,
|
||||
in which case a copy is cooperatively issued per partition of the active threads described by ``group``.
|
||||
For example, if ``group`` is a ``cooperative_groups::thread_block_tile<32, ...>``
|
||||
and the overload is called with 128 threads active, 4 copies will be issued, one cooperatively per warp.
|
||||
- If a non-group overload is called with multiple threads active,
|
||||
each thread issues its own copy and thus must have different arguments and the copies must not overlap.
|
||||
|
||||
|
||||
Template Parameters
|
||||
-------------------
|
||||
|
||||
.. list-table::
|
||||
:widths: 25 75
|
||||
:header-rows: 0
|
||||
|
||||
* - ``Group``
|
||||
- A type satisfying the [*Group*] concept.
|
||||
* - ``Shape``
|
||||
- Either `cuda::std::size_t <https://en.cppreference.com/w/c/types/size_t>`_
|
||||
or :ref:`cuda::aligned_size_t <libcudacxx-extended-api-memory-aligned-size>`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
|
||||
.. list-table::
|
||||
:widths: 25 75
|
||||
:header-rows: 0
|
||||
|
||||
* - ``group``
|
||||
- The group of threads.
|
||||
* - ``destination``
|
||||
- Pointer to the memory location to copy to.
|
||||
* - ``source``
|
||||
- Pointer to the memory location to copy from.
|
||||
* - ``size``
|
||||
- The number of bytes to copy.
|
||||
* - ``barrier``
|
||||
- The barrier object used to wait on the copy completion.
|
||||
* - ``pipeline``
|
||||
- The pipeline object used to wait on the copy completion.
|
||||
|
||||
Related traits
|
||||
--------------
|
||||
|
||||
.. code:: cuda
|
||||
|
||||
template <typename Group>
|
||||
constexpr inline bool is_thread_block_group_v;
|
||||
|
||||
This trait is ``true`` if ``Group`` represents the full CUDA thread block.
|
||||
For example, ``cooperative_groups::thread_block`` satisfies this trait.
|
||||
Users are encouraged to specialize this trait for their own groups.
|
||||
|
||||
.. code:: cuda
|
||||
|
||||
template <typename Group>
|
||||
constexpr inline bool is_warp_group_v = false;
|
||||
|
||||
This trait is ``true`` if ``Group`` represents a full CUDA warp.
|
||||
For example, ``cooperative_groups::thread_block_tile<32, ...>`` satisfies this trait.
|
||||
Users are encouraged to specialize this trait for their own groups.
|
||||
|
||||
Implementation notes
|
||||
--------------------
|
||||
|
||||
On Hopper+ GPUs, the overloads taking a barrier may use the Tensor Memory Accelerator (TMA)
|
||||
via the ``cp.async.bulk`` instruction to perform the copy if:
|
||||
- the barrier resides in shared memory,
|
||||
- the data is aligned to 16 bytes,
|
||||
- the source is global memory,
|
||||
- the destination is shared memory.
|
||||
Additionally, the cooperative overload (taking a group) can generate more efficient code
|
||||
if the group satisfies the trait ``cuda::is_thread_block_group_v`` or ``cuda::is_warp_group_v``.
|
||||
In those cases, a uniform data path is generated for the bulk copy and thread peeling is avoided.
|
||||
|
||||
On Ampere+ GPUs, the ``cp.async`` instruction may be used to perform the copy if:
|
||||
- the data is aligned to at least 4 bytes,
|
||||
- the source is global memory,
|
||||
- the destination is shared memory.
|
||||
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
.. rubric:: Example: Using a system-wide barrier to copy within global memory
|
||||
|
||||
.. code:: cuda
|
||||
|
||||
#include <cuda/barrier>
|
||||
|
||||
__global__ void example_kernel(char* dst, char* src) {
|
||||
cuda::barrier<cuda::thread_scope_system> bar;
|
||||
init(&bar, 1);
|
||||
|
||||
cuda::memcpy_async(dst, src, 1, bar);
|
||||
cuda::memcpy_async(dst + 1, src + 8, 1, bar);
|
||||
cuda::memcpy_async(dst + 2, src + 16, 1, bar);
|
||||
cuda::memcpy_async(dst + 3, src + 24, 1, bar);
|
||||
|
||||
bar.arrive_and_wait();
|
||||
}
|
||||
|
||||
`See it on Godbolt <https://godbolt.org/z/od6q9s8fq>`_
|
||||
|
||||
.. rubric:: Example: 1D load of two buffers from global to shared memory with a barrier
|
||||
|
||||
.. code:: cuda
|
||||
|
||||
#include <cuda/barrier>
|
||||
|
||||
__global__ void example_kernel(int* gmem1, double* gmem2) {
|
||||
constexpr int tile_size = 1024;
|
||||
__shared__ alignas(16) int smem1[tile_size];
|
||||
__shared__ alignas(16) double smem2[tile_size];
|
||||
|
||||
#pragma nv_diag_suppress static_var_with_dynamic_init
|
||||
__shared__ cuda::barrier<cuda::thread_scope_block> bar;
|
||||
|
||||
// setup the barrier where each thread in the block arrives at
|
||||
if (threadIdx.x == 0) {
|
||||
init(&bar, blockDim.x);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// issue two copy operations
|
||||
auto group = ...;
|
||||
cuda::memcpy_async(group, smem1, gmem1, cuda::aligned_size_t<16>(tile_size * sizeof(int) ), bar);
|
||||
cuda::memcpy_async(group, smem2, gmem2, cuda::aligned_size_t<16>(tile_size * sizeof(double)), bar);
|
||||
|
||||
// arrive and wait for copy operations to complete
|
||||
bar.arrive_and_wait();
|
||||
|
||||
// process data in smem ...
|
||||
}
|
||||
|
||||
There are multiple possibilities to initialize the ``group`` variable.
|
||||
One option is to use the cooperative groups API:
|
||||
|
||||
.. code:: cuda
|
||||
|
||||
#include <cooperative_groups.h>
|
||||
auto group = cooperative_groups::this_thread_block();
|
||||
|
||||
Another option, especially if the dimensionality of the thread block is known, e.g. 1D,
|
||||
a custom group can be defined like:
|
||||
|
||||
.. code:: cuda
|
||||
|
||||
struct this_thread_block_1D {
|
||||
static constexpr cuda::thread_scope thread_scope = cuda::thread_scope_block;
|
||||
|
||||
__device__ void sync() const {
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
__device__ auto size() const {
|
||||
return blockDim.x;
|
||||
}
|
||||
|
||||
__device__ auto thread_rank() const {
|
||||
return threadIdx.x;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
inline constexpr bool cuda::is_thread_block_group_v<this_thread_block_1D> = true;
|
||||
|
||||
Such a group will emit the least amount of code when used with ``cuda::memcpy_async``,
|
||||
since the ``thread_rank()`` is easily computed (because the block is 1D)
|
||||
and we declared the group as representing the whole thread block,
|
||||
which allows emit a uniform data path on Hopper+ GPUs in certain conditions.
|
||||
|
||||
`See it on Godbolt <https://godbolt.org/z/aM9cbabcW>`__
|
||||
@@ -0,0 +1,93 @@
|
||||
.. _libcudacxx-extended-api-asynchronous-operations-memcpy-async-tx:
|
||||
|
||||
``cuda::device::memcpy_async_tx``
|
||||
=================================
|
||||
|
||||
Defined in header ``<cuda/barrier>``:
|
||||
|
||||
.. code:: cuda
|
||||
|
||||
template <typename T, size_t Alignment>
|
||||
inline __device__
|
||||
void cuda::device::memcpy_async_tx(
|
||||
T* dest,
|
||||
const T* src,
|
||||
cuda::aligned_size_t<Alignment> size,
|
||||
cuda::barrier<cuda::thread_scope_block>& bar);
|
||||
|
||||
Copies ``size`` bytes from global memory ``src`` to shared memory ``dest`` and decrements the transaction count of ``bar`` by ``size`` bytes.
|
||||
|
||||
Preconditions
|
||||
-------------
|
||||
|
||||
- ``src``, ``dest`` are 16-byte aligned and ``size`` is a multiple of 16, i.e., ``Alignment >= 16``.
|
||||
- ``dest`` points to a shared memory allocation that is at least ``size`` bytes wide.
|
||||
- ``src`` points to a global memory allocation that is at least ``size`` bytes wide.
|
||||
- ``bar`` is located in shared memory
|
||||
- If either ``destination`` or ``source`` is an invalid or null pointer, the behavior is undefined (even if ``count`` is zero).
|
||||
|
||||
Requires
|
||||
--------
|
||||
|
||||
- ``is_trivially_copyable_v<T>`` is true.
|
||||
|
||||
Notes
|
||||
-----
|
||||
|
||||
This function can only be used under CUDA Compute Capability 9.0 (Hopper) or higher.
|
||||
|
||||
There is no feature flag to check if ``cuda::device::memcpy_async_tx`` is available.
|
||||
|
||||
**Comparison to cuda::memcpy_async**: ``memcpy_async_tx`` supports a subset of the operations of ``memcpy_async``.
|
||||
It gives more control over the synchronization with a barrier than ``memcpy_async``.
|
||||
Currently, ``memcpy_async_tx`` has no synchronous fallback mechanism., i.e., it currently does not work on older hardware
|
||||
(pre-CUDA Compute Capability 9.0, i.e., Hopper).
|
||||
|
||||
.. _libcudacxx-extended-api-asynchronous-operations-memcpy-async-tx-example:
|
||||
|
||||
Example
|
||||
-------
|
||||
|
||||
.. code:: cuda
|
||||
|
||||
#include <cuda/barrier>
|
||||
#include <cuda/std/utility> // cuda::std::move
|
||||
|
||||
#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900
|
||||
static_assert(false, "Insufficient CUDA Compute Capability: cuda::device::memcpy_async_tx is not available.");
|
||||
#endif // __CUDA_MINIMUM_ARCH__
|
||||
|
||||
__device__ alignas(16) int gmem_x[2048];
|
||||
|
||||
__device__ inline bool elect_one() {
|
||||
const unsigned int tid = threadIdx.x;
|
||||
const unsigned int warp_id = tid / 32;
|
||||
const unsigned int uniform_warp_id = __shfl_sync(0xFFFFFFFF, warp_id, 0); // broadcast from lane 0
|
||||
return (uniform_warp_id == 0 && cuda::ptx::elect_sync(0xFFFFFFFF)); // elect a leader thread among warp 0
|
||||
}
|
||||
|
||||
__global__ void example_kernel() {
|
||||
alignas(16) __shared__ int smem_x[1024];
|
||||
#pragma nv_diag_suppress static_var_with_dynamic_init
|
||||
__shared__ cuda::barrier<cuda::thread_scope_block> bar;
|
||||
|
||||
// setup the mbarrier
|
||||
if (threadIdx.x == 0) {
|
||||
init(&bar, blockDim.x);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// issue the async copy from a single thread and wait for completion
|
||||
const bool is_block_leader = elect_one();
|
||||
const int tx_count = is_block_leader ? sizeof(smem_x) : 0;
|
||||
if (is_block_leader) {
|
||||
cuda::device::memcpy_async_tx(smem_x, gmem_x, cuda::aligned_size_t<16>(tx_count), bar);
|
||||
}
|
||||
auto token = cuda::device::barrier_arrive_tx(bar, 1, tx_count);
|
||||
bar.wait(cuda::std::move(token));
|
||||
|
||||
// smem_x contains the contents of gmem_x[0], ..., gmem_x[1023]
|
||||
smem_x[threadIdx.x] += 1;
|
||||
}
|
||||
|
||||
`See it on Godbolt <https://godbolt.org/z/M8zqnrz9b>`_
|
||||
Reference in New Issue
Block a user