dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat(CCCL): device-level CUB algorithms for MoE dispatch

Browse Source 
Add complete CCCL CUB header tree (1394 files) to cccl_preload/include/:
- cub/device/ — DeviceRadixSort, DeviceScan, DeviceHistogram, DeviceReduce, DeviceSelect
- cub/agent/ — all agent implementations (sort, scan, reduce, histogram, etc)
- cub/block/ — BlockScan, BlockReduce, BlockExchange, BlockLoad, BlockStore, etc
- cub/warp/ — WarpScan, WarpReduce, WarpExchange, WarpMergeSort
- cub/thread/ — thread-level operators
- thrust/ — sort_by_key, iterator utilities
- cuda/ — execution, stream, memory_resource, functional

New kernel: cccl_moe_sort_scatter.cu
- Uses CUB DeviceRadixSort::SortPairs to sort (expert_id, token_idx) pairs
- O(n) radix sort replaces O(n log n) torch.argsort in MoE prefill path
- Boundary detection + fill for expert offsets/sizes
- Compiled against CCCL upstream headers (not corex CUB) to avoid BI-V100 bugs

Previously only 288 CCCL headers (CachingDeviceAllocator only).
Now 1394 headers — full CUB device-level algorithm stack available for
all future kernels.
This commit is contained in:
project6-dev
2026-08-13 11:18:52 +00:00
parent 7ba97f7977
commit 4c365b8c03
1108 changed files with 294533 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

197
qwen3_6_scripts/cccl_preload/include/cub/grid/grid_even_share.cuh Normal file
View File

@@ -0,0 +1,197 @@
// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved.
// SPDX-FileCopyrightText: Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved.
// SPDX-License-Identifier: BSD-3
/**
* @file
* cub::GridEvenShare is a descriptor utility for distributing input among CUDA thread blocks in an
* "even-share" fashion. Each thread block gets roughly the same number of fixed-size work units
* (grains).
*/
#pragma once
#include <cub/config.cuh>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cub/grid/grid_mapping.cuh>
#include <cub/util_math.cuh>
#include <cub/util_type.cuh>
#include <cuda/__cmath/ceil_div.h>
#include <cuda/std/__algorithm/min.h>
#include <cuda/std/limits>
CUB_NAMESPACE_BEGIN
/**
* @brief GridEvenShare is a descriptor utility for distributing input among
* CUDA thread blocks in an "even-share" fashion. Each thread block gets roughly
* the same number of input tiles.
*
* @par Overview
* Each thread block is assigned a consecutive sequence of input tiles. To help
* preserve alignment and eliminate the overhead of guarded loads for all but the
* last thread block, to GridEvenShare assigns one of three different amounts of
* work to a given thread block: "big", "normal", or "last". The "big" workloads
* are one scheduling grain larger than "normal". The "last" work unit for the
* last thread block may be partially-full if the input is not an even multiple of
* the scheduling grain size.
*
* @par
* Before invoking a child grid, a parent thread will typically construct an
* instance of GridEvenShare. The instance can be passed to child thread blocks
* which can initialize their per-thread block offsets using \p BlockInit().
*
* @rst
* .. versionadded:: 2.2.0
* First appears in CUDA Toolkit 12.3.
* @endrst
*/
template <typename OffsetT>
struct GridEvenShare
{
private:
int total_tiles{0};
int big_shares{0};
OffsetT big_share_items{0};
OffsetT normal_share_items{0};
OffsetT normal_base_offset{0};
public:
/// Total number of input items
OffsetT num_items{0};
/// Grid size in thread blocks
int grid_size{0};
/// OffsetT into input marking the beginning of the owning thread block's segment of input tiles
OffsetT block_offset{0};
/// OffsetT into input of marking the end (one-past) of the owning thread block's segment of input tiles
OffsetT block_end{0};
/// Stride between input tiles
OffsetT block_stride{0};
/**
* \brief Constructor.
*/
_CCCL_FORCEINLINE GridEvenShare() = default;
/**
* @brief Dispatch initializer. To be called prior to kernel launch.
*
* @param num_items_
* Total number of input items
*
* @param max_grid_size
* Maximum grid size allowable (actual grid size may be less if not warranted by the the
* number of input items)
*
* @param tile_items
* Number of data items per input tile
*/
_CCCL_HOST_DEVICE _CCCL_FORCEINLINE void DispatchInit(OffsetT num_items_, int max_grid_size, int tile_items)
{
if (num_items_ <= 0 || max_grid_size <= 0 || tile_items <= 0)
{
this->num_items = 0;
this->grid_size = 0;
this->block_offset = 0;
this->block_end = 0;
return;
}
this->block_offset = num_items_; // Initialize past-the-end
this->block_end = num_items_; // Initialize past-the-end
this->num_items = num_items_;
this->total_tiles = static_cast<int>(
::cuda::std::min(OffsetT{::cuda::std::numeric_limits<int>::max()}, ::cuda::ceil_div(num_items_, tile_items)));
this->grid_size = ::cuda::std::min(total_tiles, max_grid_size);
int avg_tiles_per_block = total_tiles / grid_size;
// leftover grains go to big blocks:
this->big_shares = total_tiles - (avg_tiles_per_block * grid_size);
this->normal_share_items = static_cast<OffsetT>(avg_tiles_per_block) * tile_items;
this->normal_base_offset = static_cast<OffsetT>(big_shares) * tile_items;
this->big_share_items = normal_share_items + tile_items;
}
/**
* @brief Initializes ranges for the specified thread block index. Specialized
* for a "raking" access pattern in which each thread block is assigned a
* consecutive sequence of input tiles.
*/
template <int TILE_ITEMS>
_CCCL_DEVICE _CCCL_FORCEINLINE void BlockInit(int block_id, detail::constant_t<GRID_MAPPING_RAKE> /*strategy_tag*/)
{
block_stride = TILE_ITEMS;
if (block_id < big_shares)
{
// This thread block gets a big share of grains (avg_tiles_per_block + 1)
block_offset = (block_id * big_share_items);
block_end = block_offset + big_share_items;
}
else if (block_id < total_tiles)
{
// This thread block gets a normal share of grains (avg_tiles_per_block)
block_offset = normal_base_offset + (block_id * normal_share_items);
// Avoid generating values greater than num_items, as it may cause overflow
block_end = block_offset + ::cuda::std::min(num_items - block_offset, normal_share_items);
}
// Else default past-the-end
}
/**
* @brief Block-initialization, specialized for a "raking" access
* pattern in which each thread block is assigned a consecutive sequence
* of input tiles.
*/
template <int TILE_ITEMS>
_CCCL_DEVICE _CCCL_FORCEINLINE void
BlockInit(int block_id, detail::constant_t<GRID_MAPPING_STRIP_MINE> /*strategy_tag*/)
{
block_stride = grid_size * OffsetT{TILE_ITEMS};
block_offset = block_id * OffsetT{TILE_ITEMS};
block_end = num_items;
}
/**
* @brief Block-initialization, specialized for "strip mining" access
* pattern in which the input tiles assigned to each thread block are
* separated by a stride equal to the the extent of the grid.
*/
template <int TILE_ITEMS, GridMappingStrategy STRATEGY>
_CCCL_DEVICE _CCCL_FORCEINLINE void BlockInit()
{
BlockInit<TILE_ITEMS>(blockIdx.x, detail::constant_v<STRATEGY>);
}
/**
* @brief Block-initialization, specialized for a "raking" access
* pattern in which each thread block is assigned a consecutive sequence
* of input tiles.
*
* @param[in] block_offset
* Threadblock begin offset (inclusive)
*
* @param[in] block_end
* Threadblock end offset (exclusive)
*/
template <int TILE_ITEMS, typename OffsetT1 = OffsetT>
_CCCL_DEVICE _CCCL_FORCEINLINE void BlockInit(OffsetT1 block_offset, OffsetT1 block_end)
{
this->block_offset = block_offset;
this->block_end = block_end;
this->block_stride = TILE_ITEMS;
}
};
CUB_NAMESPACE_END

82
qwen3_6_scripts/cccl_preload/include/cub/grid/grid_mapping.cuh Normal file
View File

@@ -0,0 +1,82 @@
// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved.
// SPDX-FileCopyrightText: Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved.
// SPDX-License-Identifier: BSD-3
/**
* \file
* cub::GridMappingStrategy enumerates alternative strategies for mapping constant-sized tiles of device-wide data onto
* a grid of CUDA thread blocks.
*/
#pragma once
#include <cub/config.cuh>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
CUB_NAMESPACE_BEGIN
/******************************************************************************
* Mapping policies
*****************************************************************************/
/**
* \brief cub::GridMappingStrategy enumerates alternative strategies for mapping constant-sized tiles of device-wide
* data onto a grid of CUDA thread blocks.
*/
enum GridMappingStrategy
{
/**
* \brief An a "raking" access pattern in which each thread block is
* assigned a consecutive sequence of input tiles
*
* \par Overview
* The input is evenly partitioned into \p p segments, where \p p is
* constant and corresponds loosely to the number of thread blocks that may
* actively reside on the target device. Each segment is comprised of
* consecutive tiles, where a tile is a small, constant-sized unit of input
* to be processed to completion before the thread block terminates or
* obtains more work. The kernel invokes \p p thread blocks, each
* of which iteratively consumes a segment of <em>n</em>/<em>p</em> elements
* in tile-size increments.
*/
GRID_MAPPING_RAKE,
/**
* \brief An a "strip mining" access pattern in which the input tiles assigned
* to each thread block are separated by a stride equal to the the extent of
* the grid.
*
* \par Overview
* The input is evenly partitioned into \p p sets, where \p p is
* constant and corresponds loosely to the number of thread blocks that may
* actively reside on the target device. Each set is comprised of
* data tiles separated by stride \p tiles, where a tile is a small,
* constant-sized unit of input to be processed to completion before the
* thread block terminates or obtains more work. The kernel invokes \p p
* thread blocks, each of which iteratively consumes a segment of
* <em>n</em>/<em>p</em> elements in tile-size increments.
*/
GRID_MAPPING_STRIP_MINE,
/**
* \brief A dynamic "queue-based" strategy for assigning input tiles to thread blocks.
*
* \par Overview
* The input is treated as a queue to be dynamically consumed by a grid of
* thread blocks. Work is atomically dequeued in tiles, where a tile is a
* unit of input to be processed to completion before the thread block
* terminates or obtains more work. The grid size \p p is constant,
* loosely corresponding to the number of thread blocks that may actively
* reside on the target device.
*/
GRID_MAPPING_DYNAMIC,
};
CUB_NAMESPACE_END

201
qwen3_6_scripts/cccl_preload/include/cub/grid/grid_queue.cuh Normal file
View File

@@ -0,0 +1,201 @@
// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved.
// SPDX-FileCopyrightText: Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved.
// SPDX-License-Identifier: BSD-3
/**
* @file
* cub::GridQueue is a descriptor utility for dynamic queue management.
*/
#pragma once
#include <cub/config.cuh>
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
# pragma GCC system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
# pragma clang system_header
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
# pragma system_header
#endif // no system header
#include <cub/util_debug.cuh>
#include <nv/target>
CUB_NAMESPACE_BEGIN
/**
* @brief GridQueue is a descriptor utility for dynamic queue management.
*
* @par Overview
* GridQueue descriptors provides abstractions for "filling" or
* "draining" globally-shared vectors.
*
* @par
* A "filling" GridQueue works by atomically-adding to a zero-initialized counter,
* returning a unique offset for the calling thread to write its items.
* The GridQueue maintains the total "fill-size". The fill counter must be reset
* using GridQueue::ResetFill by the host or kernel instance prior to the kernel instance that
* will be filling.
*
* @par
* Similarly, a "draining" GridQueue works by atomically-incrementing a
* zero-initialized counter, returning a unique offset for the calling thread to
* read its items. Threads can safely drain until the array's logical fill-size is
* exceeded. The drain counter must be reset using GridQueue::ResetDrain or
* GridQueue::FillAndResetDrain by the host or kernel instance prior to the kernel instance that
* will be filling. (For dynamic work distribution of existing data, the corresponding fill-size
* is simply the number of elements in the array.)
*
* @par
* Iterative work management can be implemented simply with a pair of flip-flopping
* work buffers, each with an associated set of fill and drain GridQueue descriptors.
*
* @rst
* .. versionadded:: 2.2.0
* First appears in CUDA Toolkit 12.3.
* @endrst
*
* @tparam OffsetT Signed integer type for global offsets
*/
template <typename OffsetT>
class GridQueue
{
private:
/// Counter indices
static constexpr int FILL = 0;
static constexpr int DRAIN = 1;
/// Pair of counters
OffsetT* d_counters;
public:
/// Returns the device allocation size in bytes needed to construct a GridQueue instance
_CCCL_HOST_DEVICE _CCCL_FORCEINLINE static size_t AllocationSize()
{
return sizeof(OffsetT) * 2;
}
/// Constructs an invalid GridQueue descriptor
_CCCL_HOST_DEVICE _CCCL_FORCEINLINE GridQueue()
: d_counters(nullptr)
{}
/**
* @brief Constructs a GridQueue descriptor around the device storage allocation
*
* @param d_storage
* Device allocation to back the GridQueue. Must be at least as big as
* <tt>AllocationSize()</tt>.
*/
_CCCL_HOST_DEVICE _CCCL_FORCEINLINE GridQueue(void* d_storage)
: d_counters((OffsetT*) d_storage)
{}
/// This operation sets the fill-size and resets the drain counter, preparing the GridQueue for
/// draining in the next kernel instance. To be called by the host or by a kernel prior to the one
/// which will be draining.
_CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t
FillAndResetDrain(OffsetT fill_size, [[maybe_unused]] cudaStream_t stream = nullptr)
{
cudaError_t result = cudaErrorUnknown;
NV_IF_ELSE_TARGET(
NV_IS_DEVICE,
({
d_counters[FILL] = fill_size;
d_counters[DRAIN] = 0;
result = cudaSuccess;
}),
({
OffsetT counters[2];
counters[FILL] = fill_size;
counters[DRAIN] = 0;
result = CubDebug(cudaMemcpyAsync(d_counters, counters, sizeof(OffsetT) * 2, cudaMemcpyHostToDevice, stream));
}));
return result;
}
/// This operation resets the drain so that it may advance to meet the existing fill-size.
/// To be called by the host or by a kernel prior to the one which will be draining.
_CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t ResetDrain([[maybe_unused]] cudaStream_t stream = nullptr)
{
cudaError_t result = cudaErrorUnknown;
NV_IF_ELSE_TARGET(NV_IS_DEVICE,
({
d_counters[DRAIN] = 0;
result = cudaSuccess;
}),
({ result = CubDebug(cudaMemsetAsync(d_counters + DRAIN, 0, sizeof(OffsetT), stream)); }));
return result;
}
/// This operation resets the fill counter.
/// To be called by the host or by a kernel prior to the one which will be filling.
_CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t ResetFill([[maybe_unused]] cudaStream_t stream = nullptr)
{
cudaError_t result = cudaErrorUnknown;
NV_IF_ELSE_TARGET(NV_IS_DEVICE,
({
d_counters[FILL] = 0;
result = cudaSuccess;
}),
({ result = CubDebug(cudaMemsetAsync(d_counters + FILL, 0, sizeof(OffsetT), stream)); }));
return result;
}
/// Returns the fill-size established by the parent or by the previous kernel.
_CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t
FillSize(OffsetT& fill_size, [[maybe_unused]] cudaStream_t stream = nullptr)
{
cudaError_t result = cudaErrorUnknown;
NV_IF_ELSE_TARGET(
NV_IS_DEVICE,
({
fill_size = d_counters[FILL];
result = cudaSuccess;
}),
({
result =
CubDebug(cudaMemcpyAsync(&fill_size, d_counters + FILL, sizeof(OffsetT), cudaMemcpyDeviceToHost, stream));
}));
return result;
}
/// Drain @p num_items from the queue. Returns offset from which to read items.
/// To be called from CUDA kernel.
_CCCL_DEVICE _CCCL_FORCEINLINE OffsetT Drain(OffsetT num_items)
{
return atomicAdd(d_counters + DRAIN, num_items);
}
/// Fill @p num_items into the queue. Returns offset from which to write items.
/// To be called from CUDA kernel.
_CCCL_DEVICE _CCCL_FORCEINLINE OffsetT Fill(OffsetT num_items)
{
return atomicAdd(d_counters + FILL, num_items);
}
};
#ifndef _CCCL_DOXYGEN_INVOKED // Do not document
/**
* Reset grid queue (call with 1 block of 1 thread)
*/
template <typename OffsetT>
_CCCL_KERNEL_ATTRIBUTES void FillAndResetDrainKernel(GridQueue<OffsetT> grid_queue, OffsetT num_items)
{
grid_queue.FillAndResetDrain(num_items);
}
#endif // _CCCL_DOXYGEN_INVOKED
CUB_NAMESPACE_END