4c365b8c03
Add complete CCCL CUB header tree (1394 files) to cccl_preload/include/: - cub/device/ — DeviceRadixSort, DeviceScan, DeviceHistogram, DeviceReduce, DeviceSelect - cub/agent/ — all agent implementations (sort, scan, reduce, histogram, etc) - cub/block/ — BlockScan, BlockReduce, BlockExchange, BlockLoad, BlockStore, etc - cub/warp/ — WarpScan, WarpReduce, WarpExchange, WarpMergeSort - cub/thread/ — thread-level operators - thrust/ — sort_by_key, iterator utilities - cuda/ — execution, stream, memory_resource, functional New kernel: cccl_moe_sort_scatter.cu - Uses CUB DeviceRadixSort::SortPairs to sort (expert_id, token_idx) pairs - O(n) radix sort replaces O(n log n) torch.argsort in MoE prefill path - Boundary detection + fill for expert offsets/sizes - Compiled against CCCL upstream headers (not corex CUB) to avoid BI-V100 bugs Previously only 288 CCCL headers (CachingDeviceAllocator only). Now 1394 headers — full CUB device-level algorithm stack available for all future kernels.
394 lines
12 KiB
C++
394 lines
12 KiB
C++
// SPDX-FileCopyrightText: Copyright (c) 2008-2013, NVIDIA Corporation. All rights reserved.
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
/*! \file find.h
|
|
* \brief Locating values in (unsorted) ranges
|
|
*/
|
|
|
|
#pragma once
|
|
|
|
#include <thrust/detail/config.h>
|
|
|
|
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
|
# pragma GCC system_header
|
|
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
|
# pragma clang system_header
|
|
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
|
# pragma system_header
|
|
#endif // no system header
|
|
#include <thrust/detail/execution_policy.h>
|
|
|
|
THRUST_NAMESPACE_BEGIN
|
|
|
|
/*! \addtogroup algorithms
|
|
*/
|
|
|
|
/*! \addtogroup searching
|
|
* \ingroup algorithms
|
|
* \{
|
|
*/
|
|
|
|
/*! \p find returns the first iterator \c i in the range
|
|
* <tt>[first, last)</tt> such that <tt>*i == value</tt>
|
|
* or \c last if no such iterator exists.
|
|
*
|
|
* The algorithm's execution is parallelized as determined by \p exec.
|
|
*
|
|
* \param exec The execution policy to use for parallelization.
|
|
* \param first Beginning of the sequence to search.
|
|
* \param last End of the sequence to search.
|
|
* \param value The value to find.
|
|
* \return The first iterator \c i such that <tt>*i == value</tt> or \c last.
|
|
*
|
|
* \tparam DerivedPolicy The name of the derived execution policy.
|
|
* \tparam InputIterator is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
|
* Iterator</a> and \p InputIterator's \c value_type is equality comparable to type \c T. \tparam T is a model of <a
|
|
* href="https://en.cppreference.com/w/cpp/named_req/LessThanComparable">EqualityComparable</a>.
|
|
*
|
|
* \code
|
|
* #include <thrust/find.h>
|
|
* #include <thrust/device_vector.h>
|
|
* #include <thrust/execution_policy.h>
|
|
* ...
|
|
* thrust::device_vector<int> input(4);
|
|
*
|
|
* input[0] = 0;
|
|
* input[1] = 5;
|
|
* input[2] = 3;
|
|
* input[3] = 7;
|
|
*
|
|
* thrust::device_vector<int>::iterator iter;
|
|
*
|
|
* iter = thrust::find(thrust::device, input.begin(), input.end(), 3); // returns input.first() + 2
|
|
* iter = thrust::find(thrust::device, input.begin(), input.end(), 5); // returns input.first() + 1
|
|
* iter = thrust::find(thrust::device, input.begin(), input.end(), 9); // returns input.end()
|
|
* \endcode
|
|
*
|
|
* \see find_if
|
|
* \see mismatch
|
|
*
|
|
* \verbatim embed:rst:leading-asterisk
|
|
* .. versionadded:: 2.2.0
|
|
* \endverbatim
|
|
*/
|
|
template <typename DerivedPolicy, typename InputIterator, typename T>
|
|
_CCCL_HOST_DEVICE InputIterator
|
|
find(const thrust::detail::execution_policy_base<DerivedPolicy>& exec,
|
|
InputIterator first,
|
|
InputIterator last,
|
|
const T& value);
|
|
|
|
/*! \p find returns the first iterator \c i in the range
|
|
* <tt>[first, last)</tt> such that <tt>*i == value</tt>
|
|
* or \c last if no such iterator exists.
|
|
*
|
|
* \param first Beginning of the sequence to search.
|
|
* \param last End of the sequence to search.
|
|
* \param value The value to find.
|
|
* \return The first iterator \c i such that <tt>*i == value</tt> or \c last.
|
|
*
|
|
* \tparam InputIterator is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
|
* Iterator</a> and \p InputIterator's \c value_type is equality comparable to type \c T. \tparam T is a model of <a
|
|
* href="https://en.cppreference.com/w/cpp/named_req/LessThanComparable">EqualityComparable</a>.
|
|
*
|
|
* \code
|
|
* #include <thrust/find.h>
|
|
* #include <thrust/device_vector.h>
|
|
* ...
|
|
* thrust::device_vector<int> input(4);
|
|
*
|
|
* input[0] = 0;
|
|
* input[1] = 5;
|
|
* input[2] = 3;
|
|
* input[3] = 7;
|
|
*
|
|
* thrust::device_vector<int>::iterator iter;
|
|
*
|
|
* iter = thrust::find(input.begin(), input.end(), 3); // returns input.first() + 2
|
|
* iter = thrust::find(input.begin(), input.end(), 5); // returns input.first() + 1
|
|
* iter = thrust::find(input.begin(), input.end(), 9); // returns input.end()
|
|
* \endcode
|
|
*
|
|
* \see find_if
|
|
* \see mismatch
|
|
*
|
|
* \verbatim embed:rst:leading-asterisk
|
|
* .. versionadded:: 2.2.0
|
|
* \endverbatim
|
|
*/
|
|
template <typename InputIterator, typename T>
|
|
InputIterator find(InputIterator first, InputIterator last, const T& value);
|
|
|
|
/*! \p find_if returns the first iterator \c i in the range
|
|
* <tt>[first, last)</tt> such that <tt>pred(*i)</tt> is \c true
|
|
* or \c last if no such iterator exists.
|
|
*
|
|
* The algorithm's execution is parallelized as determined by \p exec.
|
|
*
|
|
* \param exec The execution policy to use for parallelization.
|
|
* \param first Beginning of the sequence to search.
|
|
* \param last End of the sequence to search.
|
|
* \param pred A predicate used to test range elements.
|
|
* \return The first iterator \c i such that <tt>pred(*i)</tt> is \c true, or \c last.
|
|
*
|
|
* \tparam DerivedPolicy The name of the derived execution policy.
|
|
* \tparam InputIterator is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
|
* Iterator</a>. \tparam Predicate is a model of <a
|
|
* href="https://en.cppreference.com/w/cpp/concepts/predicate">Predicate</a>.
|
|
*
|
|
* \code
|
|
* #include <thrust/find.h>
|
|
* #include <thrust/device_vector.h>
|
|
* #include <thrust/execution_policy.h>
|
|
* ...
|
|
*
|
|
* struct greater_than_four
|
|
* {
|
|
* __host__ __device__
|
|
* bool operator()(int x)
|
|
* {
|
|
* return x > 4;
|
|
* }
|
|
* };
|
|
*
|
|
* struct greater_than_ten
|
|
* {
|
|
* __host__ __device__
|
|
* bool operator()(int x)
|
|
* {
|
|
* return x > 10;
|
|
* }
|
|
* };
|
|
*
|
|
* ...
|
|
* thrust::device_vector<int> input(4);
|
|
*
|
|
* input[0] = 0;
|
|
* input[1] = 5;
|
|
* input[2] = 3;
|
|
* input[3] = 7;
|
|
*
|
|
* thrust::device_vector<int>::iterator iter;
|
|
*
|
|
* iter = thrust::find_if(thrust::device, input.begin(), input.end(), greater_than_four()); // returns input.first() +
|
|
* 1
|
|
*
|
|
* iter = thrust::find_if(thrust::device, input.begin(), input.end(), greater_than_ten()); // returns input.end()
|
|
* \endcode
|
|
*
|
|
* \see find
|
|
* \see find_if_not
|
|
* \see mismatch
|
|
*
|
|
* \verbatim embed:rst:leading-asterisk
|
|
* .. versionadded:: 2.2.0
|
|
* \endverbatim
|
|
*/
|
|
template <typename DerivedPolicy, typename InputIterator, typename Predicate>
|
|
_CCCL_HOST_DEVICE InputIterator find_if(
|
|
const thrust::detail::execution_policy_base<DerivedPolicy>& exec,
|
|
InputIterator first,
|
|
InputIterator last,
|
|
Predicate pred);
|
|
|
|
/*! \p find_if returns the first iterator \c i in the range
|
|
* <tt>[first, last)</tt> such that <tt>pred(*i)</tt> is \c true
|
|
* or \c last if no such iterator exists.
|
|
*
|
|
* \param first Beginning of the sequence to search.
|
|
* \param last End of the sequence to search.
|
|
* \param pred A predicate used to test range elements.
|
|
* \return The first iterator \c i such that <tt>pred(*i)</tt> is \c true, or \c last.
|
|
*
|
|
* \tparam InputIterator is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
|
* Iterator</a>. \tparam Predicate is a model of <a
|
|
* href="https://en.cppreference.com/w/cpp/concepts/predicate">Predicate</a>.
|
|
*
|
|
* \code
|
|
* #include <thrust/find.h>
|
|
* #include <thrust/device_vector.h>
|
|
*
|
|
* struct greater_than_four
|
|
* {
|
|
* __host__ __device__
|
|
* bool operator()(int x)
|
|
* {
|
|
* return x > 4;
|
|
* }
|
|
* };
|
|
*
|
|
* struct greater_than_ten
|
|
* {
|
|
* __host__ __device__
|
|
* bool operator()(int x)
|
|
* {
|
|
* return x > 10;
|
|
* }
|
|
* };
|
|
*
|
|
* ...
|
|
* thrust::device_vector<int> input(4);
|
|
*
|
|
* input[0] = 0;
|
|
* input[1] = 5;
|
|
* input[2] = 3;
|
|
* input[3] = 7;
|
|
*
|
|
* thrust::device_vector<int>::iterator iter;
|
|
*
|
|
* iter = thrust::find_if(input.begin(), input.end(), greater_than_four()); // returns input.first() + 1
|
|
*
|
|
* iter = thrust::find_if(input.begin(), input.end(), greater_than_ten()); // returns input.end()
|
|
* \endcode
|
|
*
|
|
* \see find
|
|
* \see find_if_not
|
|
* \see mismatch
|
|
*
|
|
* \verbatim embed:rst:leading-asterisk
|
|
* .. versionadded:: 2.2.0
|
|
* \endverbatim
|
|
*/
|
|
template <typename InputIterator, typename Predicate>
|
|
InputIterator find_if(InputIterator first, InputIterator last, Predicate pred);
|
|
|
|
/*! \p find_if_not returns the first iterator \c i in the range
|
|
* <tt>[first, last)</tt> such that <tt>pred(*i)</tt> is \c false
|
|
* or \c last if no such iterator exists.
|
|
*
|
|
* The algorithm's execution is parallelized as determined by \p exec.
|
|
*
|
|
* \param exec The execution policy to use for parallelization.
|
|
* \param first Beginning of the sequence to search.
|
|
* \param last End of the sequence to search.
|
|
* \param pred A predicate used to test range elements.
|
|
* \return The first iterator \c i such that <tt>pred(*i)</tt> is \c false, or \c last.
|
|
*
|
|
* \tparam DerivedPolicy The name of the derived execution policy.
|
|
* \tparam InputIterator is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
|
* Iterator</a>. \tparam Predicate is a model of <a
|
|
* href="https://en.cppreference.com/w/cpp/concepts/predicate">Predicate</a>.
|
|
*
|
|
* \code
|
|
* #include <thrust/find.h>
|
|
* #include <thrust/device_vector.h>
|
|
* #include <thrust/execution_policy.h>
|
|
* ...
|
|
*
|
|
* struct greater_than_four
|
|
* {
|
|
* __host__ __device__
|
|
* bool operator()(int x)
|
|
* {
|
|
* return x > 4;
|
|
* }
|
|
* };
|
|
*
|
|
* struct greater_than_ten
|
|
* {
|
|
* __host__ __device__
|
|
* bool operator()(int x)
|
|
* {
|
|
* return x > 10;
|
|
* }
|
|
* };
|
|
*
|
|
* ...
|
|
* thrust::device_vector<int> input(4);
|
|
*
|
|
* input[0] = 0;
|
|
* input[1] = 5;
|
|
* input[2] = 3;
|
|
* input[3] = 7;
|
|
*
|
|
* thrust::device_vector<int>::iterator iter;
|
|
*
|
|
* iter = thrust::find_if_not(thrust::device, input.begin(), input.end(), greater_than_four()); // returns
|
|
* input.first()
|
|
*
|
|
* iter = thrust::find_if_not(thrust::device, input.begin(), input.end(), greater_than_ten()); // returns
|
|
* input.first() \endcode
|
|
*
|
|
* \see find
|
|
* \see find_if
|
|
* \see mismatch
|
|
*
|
|
* \verbatim embed:rst:leading-asterisk
|
|
* .. versionadded:: 2.2.0
|
|
* \endverbatim
|
|
*/
|
|
template <typename DerivedPolicy, typename InputIterator, typename Predicate>
|
|
_CCCL_HOST_DEVICE InputIterator find_if_not(
|
|
const thrust::detail::execution_policy_base<DerivedPolicy>& exec,
|
|
InputIterator first,
|
|
InputIterator last,
|
|
Predicate pred);
|
|
|
|
/*! \p find_if_not returns the first iterator \c i in the range
|
|
* <tt>[first, last)</tt> such that <tt>pred(*i)</tt> is \c false
|
|
* or \c last if no such iterator exists.
|
|
*
|
|
* \param first Beginning of the sequence to search.
|
|
* \param last End of the sequence to search.
|
|
* \param pred A predicate used to test range elements.
|
|
* \return The first iterator \c i such that <tt>pred(*i)</tt> is \c false, or \c last.
|
|
*
|
|
* \tparam InputIterator is a model of <a href="https://en.cppreference.com/w/cpp/iterator/input_iterator">Input
|
|
* Iterator</a>. \tparam Predicate is a model of <a
|
|
* href="https://en.cppreference.com/w/cpp/concepts/predicate">Predicate</a>.
|
|
*
|
|
* \code
|
|
* #include <thrust/find.h>
|
|
* #include <thrust/device_vector.h>
|
|
*
|
|
* struct greater_than_four
|
|
* {
|
|
* __host__ __device__
|
|
* bool operator()(int x)
|
|
* {
|
|
* return x > 4;
|
|
* }
|
|
* };
|
|
*
|
|
* struct greater_than_ten
|
|
* {
|
|
* __host__ __device__
|
|
* bool operator()(int x)
|
|
* {
|
|
* return x > 10;
|
|
* }
|
|
* };
|
|
*
|
|
* ...
|
|
* thrust::device_vector<int> input(4);
|
|
*
|
|
* input[0] = 0;
|
|
* input[1] = 5;
|
|
* input[2] = 3;
|
|
* input[3] = 7;
|
|
*
|
|
* thrust::device_vector<int>::iterator iter;
|
|
*
|
|
* iter = thrust::find_if_not(input.begin(), input.end(), greater_than_four()); // returns input.first()
|
|
*
|
|
* iter = thrust::find_if_not(input.begin(), input.end(), greater_than_ten()); // returns input.first()
|
|
* \endcode
|
|
*
|
|
* \see find
|
|
* \see find_if
|
|
* \see mismatch
|
|
*
|
|
* \verbatim embed:rst:leading-asterisk
|
|
* .. versionadded:: 2.2.0
|
|
* \endverbatim
|
|
*/
|
|
template <typename InputIterator, typename Predicate>
|
|
InputIterator find_if_not(InputIterator first, InputIterator last, Predicate pred);
|
|
|
|
/*! \} // end searching
|
|
*/
|
|
|
|
THRUST_NAMESPACE_END
|
|
|
|
#include <thrust/detail/find.inl>
|