feat(CCCL): device-level CUB algorithms for MoE dispatch
Add complete CCCL CUB header tree (1394 files) to cccl_preload/include/: - cub/device/ — DeviceRadixSort, DeviceScan, DeviceHistogram, DeviceReduce, DeviceSelect - cub/agent/ — all agent implementations (sort, scan, reduce, histogram, etc) - cub/block/ — BlockScan, BlockReduce, BlockExchange, BlockLoad, BlockStore, etc - cub/warp/ — WarpScan, WarpReduce, WarpExchange, WarpMergeSort - cub/thread/ — thread-level operators - thrust/ — sort_by_key, iterator utilities - cuda/ — execution, stream, memory_resource, functional New kernel: cccl_moe_sort_scatter.cu - Uses CUB DeviceRadixSort::SortPairs to sort (expert_id, token_idx) pairs - O(n) radix sort replaces O(n log n) torch.argsort in MoE prefill path - Boundary detection + fill for expert offsets/sizes - Compiled against CCCL upstream headers (not corex CUB) to avoid BI-V100 bugs Previously only 288 CCCL headers (CachingDeviceAllocator only). Now 1394 headers — full CUB device-level algorithm stack available for all future kernels.
This commit is contained in:
project6-dev
380
qwen3_6_scripts/cccl_preload/include/thrust/mismatch.h
Normal file
380
qwen3_6_scripts/cccl_preload/include/thrust/mismatch.h
Normal file
@@ -0,0 +1,380 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2008-2013, NVIDIA Corporation. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
/*! \file mismatch.h
|
||||
* \brief Search for differences between ranges
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <thrust/detail/config.h>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
#include <thrust/detail/execution_policy.h>
|
||||
|
||||
#include <cuda/std/__utility/pair.h>
|
||||
|
||||
THRUST_NAMESPACE_BEGIN
|
||||
|
||||
/*! \addtogroup algorithms
|
||||
*/
|
||||
|
||||
/*! \addtogroup searching
|
||||
* \ingroup algorithms
|
||||
* \{
|
||||
*/
|
||||
|
||||
/*! \p mismatch finds the first position where the two ranges <tt>[first1, last1)</tt>
|
||||
* and <tt>[first2, first2 + (last1 - first1))</tt> differ. The two versions of
|
||||
* \p mismatch use different tests for whether elements differ.
|
||||
*
|
||||
* This version of \p mismatch finds the first iterator \c i in <tt>[first1, last1)</tt>
|
||||
* such that <tt>*i == *(first2 + (i - first1))</tt> is \c false. The return value is a
|
||||
* \c pair whose first element is \c i and whose second element is <tt>*(first2 + (i - first1))</tt>.
|
||||
* If no such iterator \c i exists, the return value is a \c pair whose first element
|
||||
* is \c last1 and whose second element is <tt>*(first2 + (last1 - first1))</tt>.
|
||||
*
|
||||
* The algorithm's execution is parallelized as determined by \p exec.
|
||||
*
|
||||
* \param exec The execution policy to use for parallelization.
|
||||
* \param first1 The beginning of the first sequence.
|
||||
* \param last1 The end of the first sequence.
|
||||
* \param first2 The beginning of the second sequence.
|
||||
* \return The first position where the sequences differ.
|
||||
*
|
||||
* \tparam DerivedPolicy The name of the derived execution policy.
|
||||
* \tparam InputIterator1 is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
||||
* Iterator</a> and \p InputIterator1's \c value_type is equality comparable to \p InputIterator2's \c value_type.
|
||||
* \tparam InputIterator2 is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
||||
* Iterator</a>.
|
||||
*
|
||||
* \code
|
||||
* #include <thrust/mismatch.h>
|
||||
* #include <thrust/device_vector.h>
|
||||
* #include <thrust/execution_policy.h>
|
||||
* ...
|
||||
* thrust::device_vector<int> vec1(4);
|
||||
* thrust::device_vector<int> vec2(4);
|
||||
*
|
||||
* vec1[0] = 0; vec2[0] = 0;
|
||||
* vec1[1] = 5; vec2[1] = 5;
|
||||
* vec1[2] = 3; vec2[2] = 8;
|
||||
* vec1[3] = 7; vec2[3] = 7;
|
||||
*
|
||||
* using Iterator = thrust::device_vector<int>::iterator;
|
||||
* cuda::std::pair<Iterator,Iterator> result;
|
||||
*
|
||||
* result = thrust::mismatch(thrust::device, vec1.begin(), vec1.end(), vec2.begin());
|
||||
*
|
||||
* // result.first is vec1.begin() + 2
|
||||
* // result.second is vec2.begin() + 2
|
||||
* \endcode
|
||||
*
|
||||
* \see find
|
||||
* \see find_if
|
||||
*
|
||||
* \verbatim embed:rst:leading-asterisk
|
||||
* .. versionadded:: 2.2.0
|
||||
* \endverbatim
|
||||
*/
|
||||
template <typename DerivedPolicy, typename InputIterator1, typename InputIterator2>
|
||||
_CCCL_HOST_DEVICE ::cuda::std::pair<InputIterator1, InputIterator2>
|
||||
mismatch(const thrust::detail::execution_policy_base<DerivedPolicy>& exec,
|
||||
InputIterator1 first1,
|
||||
InputIterator1 last1,
|
||||
InputIterator2 first2);
|
||||
|
||||
/*! \p mismatch finds the first position where the two ranges <tt>[first1, last1)</tt>
|
||||
* and <tt>[first2, first2 + (last1 - first1))</tt> differ. The two versions of
|
||||
* \p mismatch use different tests for whether elements differ.
|
||||
*
|
||||
* This version of \p mismatch finds the first iterator \c i in <tt>[first1, last1)</tt>
|
||||
* such that <tt>*i == *(first2 + (i - first1))</tt> is \c false. The return value is a
|
||||
* \c pair whose first element is \c i and whose second element is <tt>*(first2 + (i - first1))</tt>.
|
||||
* If no such iterator \c i exists, the return value is a \c pair whose first element
|
||||
* is \c last1 and whose second element is <tt>*(first2 + (last1 - first1))</tt>.
|
||||
*
|
||||
* \param first1 The beginning of the first sequence.
|
||||
* \param last1 The end of the first sequence.
|
||||
* \param first2 The beginning of the second sequence.
|
||||
* \return The first position where the sequences differ.
|
||||
*
|
||||
* \tparam InputIterator1 is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
||||
* Iterator</a> and \p InputIterator1's \c value_type is equality comparable to \p InputIterator2's \c value_type.
|
||||
* \tparam InputIterator2 is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
||||
* Iterator</a>.
|
||||
*
|
||||
* \code
|
||||
* #include <thrust/mismatch.h>
|
||||
* #include <thrust/device_vector.h>
|
||||
* ...
|
||||
* thrust::device_vector<int> vec1(4);
|
||||
* thrust::device_vector<int> vec2(4);
|
||||
*
|
||||
* vec1[0] = 0; vec2[0] = 0;
|
||||
* vec1[1] = 5; vec2[1] = 5;
|
||||
* vec1[2] = 3; vec2[2] = 8;
|
||||
* vec1[3] = 7; vec2[3] = 7;
|
||||
*
|
||||
* using Iterator = thrust::device_vector<int>::iterator;
|
||||
* cuda::std::pair<Iterator,Iterator> result;
|
||||
*
|
||||
* result = thrust::mismatch(vec1.begin(), vec1.end(), vec2.begin());
|
||||
*
|
||||
* // result.first is vec1.begin() + 2
|
||||
* // result.second is vec2.begin() + 2
|
||||
* \endcode
|
||||
*
|
||||
* \see find
|
||||
* \see find_if
|
||||
*
|
||||
* \verbatim embed:rst:leading-asterisk
|
||||
* .. versionadded:: 2.2.0
|
||||
* \endverbatim
|
||||
*/
|
||||
template <typename InputIterator1, typename InputIterator2>
|
||||
::cuda::std::pair<InputIterator1, InputIterator2>
|
||||
mismatch(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2);
|
||||
|
||||
/*! \p mismatch finds the first position where the two ranges <tt>[first1, last1)</tt>
|
||||
* and <tt>[first2, first2 + (last1 - first1))</tt> differ. The two versions of
|
||||
* \p mismatch use different tests for whether elements differ.
|
||||
*
|
||||
* This version of \p mismatch finds the first iterator \c i in <tt>[first1, last1)</tt>
|
||||
* such that <tt>pred(\*i, \*(first2 + (i - first1))</tt> is \c false. The return value is a
|
||||
* \c pair whose first element is \c i and whose second element is <tt>*(first2 + (i - first1))</tt>.
|
||||
* If no such iterator \c i exists, the return value is a \c pair whose first element is
|
||||
* \c last1 and whose second element is <tt>*(first2 + (last1 - first1))</tt>.
|
||||
*
|
||||
* The algorithm's execution is parallelized as determined by \p exec.
|
||||
*
|
||||
* \param exec The execution policy to use for parallelization.
|
||||
* \param first1 The beginning of the first sequence.
|
||||
* \param last1 The end of the first sequence.
|
||||
* \param first2 The beginning of the second sequence.
|
||||
* \param pred The binary predicate to compare elements.
|
||||
* \return The first position where the sequences differ.
|
||||
*
|
||||
* \tparam DerivedPolicy The name of the derived execution policy.
|
||||
* \tparam InputIterator1 is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
||||
* Iterator</a>. \tparam InputIterator2 is a model of <a
|
||||
* href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input Iterator</a>. \tparam Predicate is a model of
|
||||
* <a href="https://en.cppreference.com/w/cpp/named_req/BinaryPredicate">Input Iterator</a>.
|
||||
*
|
||||
* \code
|
||||
* #include <thrust/mismatch.h>
|
||||
* #include <thrust/device_vector.h>
|
||||
* #include <thrust/execution_policy.h>
|
||||
* ...
|
||||
* thrust::device_vector<int> vec1(4);
|
||||
* thrust::device_vector<int> vec2(4);
|
||||
*
|
||||
* vec1[0] = 0; vec2[0] = 0;
|
||||
* vec1[1] = 5; vec2[1] = 5;
|
||||
* vec1[2] = 3; vec2[2] = 8;
|
||||
* vec1[3] = 7; vec2[3] = 7;
|
||||
*
|
||||
* using Iterator = thrust::device_vector<int>::iterator;
|
||||
* cuda::std::pair<Iterator,Iterator> result;
|
||||
*
|
||||
* result = thrust::mismatch(thrust::device, vec1.begin(), vec1.end(), vec2.begin(), ::cuda::std::equal_to<int>());
|
||||
*
|
||||
* // result.first is vec1.begin() + 2
|
||||
* // result.second is vec2.begin() + 2
|
||||
* \endcode
|
||||
*
|
||||
* \see find
|
||||
* \see find_if
|
||||
*
|
||||
* \verbatim embed:rst:leading-asterisk
|
||||
* .. versionadded:: 2.2.0
|
||||
* \endverbatim
|
||||
*/
|
||||
template <typename DerivedPolicy, typename InputIterator1, typename InputIterator2, typename BinaryPredicate>
|
||||
_CCCL_HOST_DEVICE ::cuda::std::pair<InputIterator1, InputIterator2> mismatch(
|
||||
const thrust::detail::execution_policy_base<DerivedPolicy>& exec,
|
||||
InputIterator1 first1,
|
||||
InputIterator1 last1,
|
||||
InputIterator2 first2,
|
||||
BinaryPredicate pred);
|
||||
|
||||
/*! \p mismatch finds the first position where the two ranges <tt>[first1, last1)</tt>
|
||||
* and <tt>[first2, first2 + (last1 - first1))</tt> differ. The two versions of
|
||||
* \p mismatch use different tests for whether elements differ.
|
||||
*
|
||||
* This version of \p mismatch finds the first iterator \c i in <tt>[first1, last1)</tt>
|
||||
* such that <tt>pred(\*i, \*(first2 + (i - first1))</tt> is \c false. The return value is a
|
||||
* \c pair whose first element is \c i and whose second element is <tt>*(first2 + (i - first1))</tt>.
|
||||
* If no such iterator \c i exists, the return value is a \c pair whose first element is
|
||||
* \c last1 and whose second element is <tt>*(first2 + (last1 - first1))</tt>.
|
||||
*
|
||||
* \param first1 The beginning of the first sequence.
|
||||
* \param last1 The end of the first sequence.
|
||||
* \param first2 The beginning of the second sequence.
|
||||
* \param pred The binary predicate to compare elements.
|
||||
* \return The first position where the sequences differ.
|
||||
*
|
||||
* \tparam InputIterator1 is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
||||
* Iterator</a>. \tparam InputIterator2 is a model of <a
|
||||
* href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input Iterator</a>. \tparam Predicate is a model of
|
||||
* <a href="https://en.cppreference.com/w/cpp/named_req/BinaryPredicate">Input Iterator</a>.
|
||||
*
|
||||
* \code
|
||||
* #include <thrust/mismatch.h>
|
||||
* #include <thrust/device_vector.h>
|
||||
* ...
|
||||
* thrust::device_vector<int> vec1(4);
|
||||
* thrust::device_vector<int> vec2(4);
|
||||
*
|
||||
* vec1[0] = 0; vec2[0] = 0;
|
||||
* vec1[1] = 5; vec2[1] = 5;
|
||||
* vec1[2] = 3; vec2[2] = 8;
|
||||
* vec1[3] = 7; vec2[3] = 7;
|
||||
*
|
||||
* using Iterator = thrust::device_vector<int>::iterator;
|
||||
* cuda::std::pair<Iterator,Iterator> result;
|
||||
*
|
||||
* result = thrust::mismatch(vec1.begin(), vec1.end(), vec2.begin(), ::cuda::std::equal_to<int>());
|
||||
*
|
||||
* // result.first is vec1.begin() + 2
|
||||
* // result.second is vec2.begin() + 2
|
||||
* \endcode
|
||||
*
|
||||
* \see find
|
||||
* \see find_if
|
||||
*
|
||||
* \verbatim embed:rst:leading-asterisk
|
||||
* .. versionadded:: 2.2.0
|
||||
* \endverbatim
|
||||
*/
|
||||
template <typename InputIterator1, typename InputIterator2, typename BinaryPredicate>
|
||||
::cuda::std::pair<InputIterator1, InputIterator2>
|
||||
mismatch(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, BinaryPredicate pred);
|
||||
|
||||
/*! \p mismatch finds the first position where the two ranges <tt>[first1, last1)</tt>
|
||||
* and <tt>[first2, last2)</tt> differ. This bounded overload stops as soon as either
|
||||
* range is exhausted, preventing reads past the end of the shorter range.
|
||||
*
|
||||
* The algorithm's execution is parallelized as determined by \p exec.
|
||||
*
|
||||
* \param exec The execution policy to use for parallelization.
|
||||
* \param first1 The beginning of the first sequence.
|
||||
* \param last1 The end of the first sequence.
|
||||
* \param first2 The beginning of the second sequence.
|
||||
* \param last2 The end of the second sequence.
|
||||
* \return The first position where the sequences differ, or the end of the
|
||||
* shorter range if they match up to that point.
|
||||
*
|
||||
* \tparam DerivedPolicy The name of the derived execution policy.
|
||||
* \tparam InputIterator1 is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
||||
* Iterator</a> and \p InputIterator1's \c value_type is equality comparable to \p InputIterator2's \c value_type.
|
||||
* \tparam InputIterator2 is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
||||
* Iterator</a>.
|
||||
*
|
||||
* \verbatim embed:rst:leading-asterisk
|
||||
* .. versionadded:: 3.4.0
|
||||
* \endverbatim
|
||||
*/
|
||||
template <typename DerivedPolicy, typename InputIterator1, typename InputIterator2>
|
||||
_CCCL_HOST_DEVICE ::cuda::std::pair<InputIterator1, InputIterator2> mismatch(
|
||||
const thrust::detail::execution_policy_base<DerivedPolicy>& exec,
|
||||
InputIterator1 first1,
|
||||
InputIterator1 last1,
|
||||
InputIterator2 first2,
|
||||
InputIterator2 last2);
|
||||
|
||||
/*! \p mismatch finds the first position where the two ranges <tt>[first1, last1)</tt>
|
||||
* and <tt>[first2, last2)</tt> differ. This bounded overload stops as soon as either
|
||||
* range is exhausted, preventing reads past the end of the shorter range.
|
||||
*
|
||||
* \param first1 The beginning of the first sequence.
|
||||
* \param last1 The end of the first sequence.
|
||||
* \param first2 The beginning of the second sequence.
|
||||
* \param last2 The end of the second sequence.
|
||||
* \return The first position where the sequences differ, or the end of the
|
||||
* shorter range if they match up to that point.
|
||||
*
|
||||
* \tparam InputIterator1 is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
||||
* Iterator</a> and \p InputIterator1's \c value_type is equality comparable to \p InputIterator2's \c value_type.
|
||||
* \tparam InputIterator2 is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
||||
* Iterator</a>.
|
||||
*
|
||||
* \verbatim embed:rst:leading-asterisk
|
||||
* .. versionadded:: 3.4.0
|
||||
* \endverbatim
|
||||
*/
|
||||
template <typename InputIterator1, typename InputIterator2>
|
||||
::cuda::std::pair<InputIterator1, InputIterator2>
|
||||
mismatch(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2);
|
||||
|
||||
/*! \p mismatch finds the first position where the two ranges <tt>[first1, last1)</tt>
|
||||
* and <tt>[first2, last2)</tt> differ. This bounded overload stops as soon as either
|
||||
* range is exhausted, preventing reads past the end of the shorter range.
|
||||
*
|
||||
* The algorithm's execution is parallelized as determined by \p exec.
|
||||
*
|
||||
* \param exec The execution policy to use for parallelization.
|
||||
* \param first1 The beginning of the first sequence.
|
||||
* \param last1 The end of the first sequence.
|
||||
* \param first2 The beginning of the second sequence.
|
||||
* \param last2 The end of the second sequence.
|
||||
* \param pred The binary predicate to compare elements.
|
||||
* \return The first position where the sequences differ, or the end of the
|
||||
* shorter range if they match up to that point.
|
||||
*
|
||||
* \tparam DerivedPolicy The name of the derived execution policy.
|
||||
* \tparam InputIterator1 is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
||||
* Iterator</a>. \tparam InputIterator2 is a model of <a
|
||||
* href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input Iterator</a>. \tparam BinaryPredicate is a
|
||||
* model of <a href="https://en.cppreference.com/w/cpp/named_req/BinaryPredicate">Binary Predicate</a>.
|
||||
*
|
||||
* \verbatim embed:rst:leading-asterisk
|
||||
* .. versionadded:: 3.4.0
|
||||
* \endverbatim
|
||||
*/
|
||||
template <typename DerivedPolicy, typename InputIterator1, typename InputIterator2, typename BinaryPredicate>
|
||||
_CCCL_HOST_DEVICE ::cuda::std::pair<InputIterator1, InputIterator2> mismatch(
|
||||
const thrust::detail::execution_policy_base<DerivedPolicy>& exec,
|
||||
InputIterator1 first1,
|
||||
InputIterator1 last1,
|
||||
InputIterator2 first2,
|
||||
InputIterator2 last2,
|
||||
BinaryPredicate pred);
|
||||
|
||||
/*! \p mismatch finds the first position where the two ranges <tt>[first1, last1)</tt>
|
||||
* and <tt>[first2, last2)</tt> differ. This bounded overload stops as soon as either
|
||||
* range is exhausted, preventing reads past the end of the shorter range.
|
||||
*
|
||||
* \param first1 The beginning of the first sequence.
|
||||
* \param last1 The end of the first sequence.
|
||||
* \param first2 The beginning of the second sequence.
|
||||
* \param last2 The end of the second sequence.
|
||||
* \param pred The binary predicate to compare elements.
|
||||
* \return The first position where the sequences differ, or the end of the
|
||||
* shorter range if they match up to that point.
|
||||
*
|
||||
* \tparam InputIterator1 is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
||||
* Iterator</a>. \tparam InputIterator2 is a model of <a
|
||||
* href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input Iterator</a>. \tparam BinaryPredicate is a
|
||||
* model of <a href="https://en.cppreference.com/w/cpp/named_req/BinaryPredicate">Binary Predicate</a>.
|
||||
*
|
||||
* \verbatim embed:rst:leading-asterisk
|
||||
* .. versionadded:: 3.4.0
|
||||
* \endverbatim
|
||||
*/
|
||||
template <typename InputIterator1, typename InputIterator2, typename BinaryPredicate>
|
||||
::cuda::std::pair<InputIterator1, InputIterator2> mismatch(
|
||||
InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2, BinaryPredicate pred);
|
||||
|
||||
/*! \} // end searching
|
||||
*/
|
||||
|
||||
THRUST_NAMESPACE_END
|
||||
|
||||
#include <thrust/detail/mismatch.inl>
|
||||
Reference in New Issue
Block a user