4c365b8c03
Add complete CCCL CUB header tree (1394 files) to cccl_preload/include/: - cub/device/ — DeviceRadixSort, DeviceScan, DeviceHistogram, DeviceReduce, DeviceSelect - cub/agent/ — all agent implementations (sort, scan, reduce, histogram, etc) - cub/block/ — BlockScan, BlockReduce, BlockExchange, BlockLoad, BlockStore, etc - cub/warp/ — WarpScan, WarpReduce, WarpExchange, WarpMergeSort - cub/thread/ — thread-level operators - thrust/ — sort_by_key, iterator utilities - cuda/ — execution, stream, memory_resource, functional New kernel: cccl_moe_sort_scatter.cu - Uses CUB DeviceRadixSort::SortPairs to sort (expert_id, token_idx) pairs - O(n) radix sort replaces O(n log n) torch.argsort in MoE prefill path - Boundary detection + fill for expert offsets/sizes - Compiled against CCCL upstream headers (not corex CUB) to avoid BI-V100 bugs Previously only 288 CCCL headers (CachingDeviceAllocator only). Now 1394 headers — full CUB device-level algorithm stack available for all future kernels.
121 lines
4.2 KiB
C++
121 lines
4.2 KiB
C++
// SPDX-FileCopyrightText: Copyright (c) 2008-2013, NVIDIA Corporation. All rights reserved.
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
/*! \file tabulate.h
|
|
* \brief Fills a range with the tabulation of a function
|
|
*/
|
|
|
|
#pragma once
|
|
|
|
#include <thrust/detail/config.h>
|
|
|
|
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
|
# pragma GCC system_header
|
|
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
|
# pragma clang system_header
|
|
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
|
# pragma system_header
|
|
#endif // no system header
|
|
#include <thrust/detail/execution_policy.h>
|
|
|
|
THRUST_NAMESPACE_BEGIN
|
|
|
|
/*! \addtogroup transformations
|
|
* \{
|
|
*/
|
|
|
|
/*! \p tabulate fills the range <tt>[first, last)</tt> with the value of a function applied to each
|
|
* element's index.
|
|
*
|
|
* For each iterator \c i in the range <tt>[first, last)</tt>, \p tabulate performs the assignment
|
|
* <tt>*i = unary_op(i - first)</tt>.
|
|
*
|
|
* The algorithm's execution is parallelized as determined by \p exec.
|
|
*
|
|
* \param exec The execution policy to use for parallelization.
|
|
* \param first The beginning of the range.
|
|
* \param last The end of the range.
|
|
* \param unary_op The unary operation to apply.
|
|
*
|
|
* \tparam DerivedPolicy The name of the derived execution policy.
|
|
* \tparam ForwardIterator is a model of <a href="https://en.cppreference.com/w/cpp/iterator/forward_iterator">Forward
|
|
* Iterator</a>, and \p ForwardIterator is mutable, and if \c x and \c y are objects of \c ForwardIterator's \c
|
|
* value_type, then <tt>x + y</tt> is defined, and if \c T is \p ForwardIterator's \c value_type, then <tt>T(0)</tt> is
|
|
* defined.
|
|
* \tparam UnaryOperation The function's return type must be convertible to \c OutputIterator's \c value_type.
|
|
*
|
|
* The following code snippet demonstrates how to use \p tabulate to generate the first \c n non-positive integers
|
|
* using the \p thrust::host execution policy for parallelization:
|
|
*
|
|
* \code
|
|
* #include <thrust/tabulate.h>
|
|
* #include <thrust/functional.h>
|
|
* #include <thrust/execution_policy.h>
|
|
* ...
|
|
* const int N = 10;
|
|
* int A[N];
|
|
* thrust::tabulate(thrust::host, A, A + 10, ::cuda::std::negate<int>());
|
|
* // A is now {0, -1, -2, -3, -4, -5, -6, -7, -8, -9}
|
|
* \endcode
|
|
*
|
|
* \see thrust::fill
|
|
* \see thrust::generate
|
|
* \see thrust::sequence
|
|
*
|
|
* \verbatim embed:rst:leading-asterisk
|
|
* .. versionadded:: 2.2.0
|
|
* \endverbatim
|
|
*/
|
|
template <typename DerivedPolicy, typename ForwardIterator, typename UnaryOperation>
|
|
_CCCL_HOST_DEVICE void
|
|
tabulate(const thrust::detail::execution_policy_base<DerivedPolicy>& exec,
|
|
ForwardIterator first,
|
|
ForwardIterator last,
|
|
UnaryOperation unary_op);
|
|
|
|
/*! \p tabulate fills the range <tt>[first, last)</tt> with the value of a function applied to each
|
|
* element's index.
|
|
*
|
|
* For each iterator \c i in the range <tt>[first, last)</tt>, \p tabulate performs the assignment
|
|
* <tt>*i = unary_op(i - first)</tt>.
|
|
*
|
|
* \param first The beginning of the range.
|
|
* \param last The end of the range.
|
|
* \param unary_op The unary operation to apply.
|
|
*
|
|
* \tparam ForwardIterator is a model of <a href="https://en.cppreference.com/w/cpp/iterator/forward_iterator">Forward
|
|
* Iterator</a>, and \p ForwardIterator is mutable, and if \c x and \c y are objects of \c ForwardIterator's \c
|
|
* value_type, then <tt>x + y</tt> is defined, and if \c T is \p ForwardIterator's \c value_type, then <tt>T(0)</tt> is
|
|
* defined.
|
|
* \tparam UnaryOperation The function's return type must be convertible to \c OutputIterator's \c value_type.
|
|
*
|
|
* The following code snippet demonstrates how to use \p tabulate to generate the first \c n non-positive integers:
|
|
*
|
|
* \code
|
|
* #include <thrust/tabulate.h>
|
|
* #include <thrust/functional.h>
|
|
* ...
|
|
* const int N = 10;
|
|
* int A[N];
|
|
* thrust::tabulate(A, A + 10, ::cuda::std::negate<int>());
|
|
* // A is now {0, -1, -2, -3, -4, -5, -6, -7, -8, -9}
|
|
* \endcode
|
|
*
|
|
* \see thrust::fill
|
|
* \see thrust::generate
|
|
* \see thrust::sequence
|
|
*
|
|
* \verbatim embed:rst:leading-asterisk
|
|
* .. versionadded:: 2.2.0
|
|
* \endverbatim
|
|
*/
|
|
template <typename ForwardIterator, typename UnaryOperation>
|
|
void tabulate(ForwardIterator first, ForwardIterator last, UnaryOperation unary_op);
|
|
|
|
/*! \} // end transformations
|
|
*/
|
|
|
|
THRUST_NAMESPACE_END
|
|
|
|
#include <thrust/detail/tabulate.inl>
|