c7a63bc2c8
Fixes found by reading all 17 muh files + 6 CCCL counterpart policy_selectors as full source code input: 1. topk: BLOCK_LOAD_DIRECT → BLOCK_LOAD_VECTORIZE (CCCL SM90+ uses VECTORIZE). bits_per_pass was wrong (muh: ks<=4→9, CCCL: ks>=2→11). items now computed dynamically (4*4/key_size) not hardcoded. 2. reduce: added determinism dispatch — three modes matching CCCL: gpu_to_gpu (BLOCK_REDUCE_RAKING, vec_size=1, LOAD_DEFAULT), run_to_run (WARP_REDUCTIONS, LOAD_LDG, default), not_guaranteed (WARP_REDUCTIONS_NONDETERMINISTIC). Added bi100_det_float32 and bi100_det_float64 tuning structs with SM90 benchmark reference values. 3. batch_memcpy: flat single-tier → SmallBuffer+LargeBuffer two-tier matching CCCL structure (128 threads small, 256 threads large, warp_threshold=128, block_threshold=8192). 4. transform: single BulkPolicy → three-policy structure (VectorizedPolicy + AsyncCopyPolicy + PrefetchPolicy) matching CCCL. items_per_thread computed from bytes_in_flight / (threads * elem_size). 5. compile_test: 17 checks → 33 checks. Now verifies exact values: reduce determinism modes, topk VECTORIZE + bits=11, batch_memcpy two-tier thresholds, transform three-policy structure. 6. gen_patch: added fallback extraction for inline policy_selector values (topk now generates SAMPLING_BLOCK_SIZE patch). 7. MUH_PROJECT_CHECKPOINT.md: 'PRD设计阶段还没有代码' → actual status. 7 files changed, 413 insertions, 265 deletions.
87 lines
2.8 KiB
Plaintext
87 lines
2.8 KiB
Plaintext
// muh/include/muh/tuning/tuning_batch_memcpy.cuh — BI-V100 batch memcpy tuning
|
||
//
|
||
// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_batch_memcpy.cuh
|
||
//
|
||
// vllm impact: KV cache block copy between GPU memory regions
|
||
// Competition weight: Cache TPS × 0.56
|
||
//
|
||
// CCCL structure: two-tier (SmallBuffer handled by single block,
|
||
// LargeBuffer by multi-block collaboration). Thresholds:
|
||
// warp_level: 128 bytes
|
||
// block_level: 8 KiB
|
||
// muh must replicate this structure, not flatten it.
|
||
|
||
#pragma once
|
||
|
||
#include "muh/hardware.cuh"
|
||
#include "muh/tuning/common.cuh"
|
||
|
||
namespace muh::tuning::batch_memcpy {
|
||
|
||
/// Small buffer policy: single thread block handles many small buffers
|
||
struct SmallBufferPolicy {
|
||
int threads_per_block;
|
||
int buffers_per_thread;
|
||
int bytes_per_thread;
|
||
bool prefer_pow2_bits;
|
||
int block_level_tile_size;
|
||
int warp_level_threshold;
|
||
int block_level_threshold;
|
||
LookbackDelayPolicy buffer_lookback_delay;
|
||
LookbackDelayPolicy block_lookback_delay;
|
||
};
|
||
|
||
/// Large buffer policy: multiple blocks collaborate on one large buffer
|
||
struct LargeBufferPolicy {
|
||
int threads_per_block;
|
||
int bytes_per_thread;
|
||
};
|
||
|
||
/// Full batch memcpy policy
|
||
struct BatchMemcpyPolicy {
|
||
SmallBufferPolicy small_buffer;
|
||
LargeBufferPolicy large_buffer;
|
||
};
|
||
|
||
// ============================================================
|
||
// BI-V100 tuning values — from CCCL SM70+ defaults
|
||
//
|
||
// CCCL policy_selector (all architectures):
|
||
// small: 128 threads, 4 bufs/thread, 8 bytes/thread
|
||
// prefer_pow2_bits = (cc < 7.0)
|
||
// warp_threshold = 128, block_threshold = 8192
|
||
// delays = default_delay_constructor_policy(true)
|
||
// large: 256 threads, 32 bytes/thread
|
||
//
|
||
// For BI-V100: start with CCCL defaults.
|
||
// The delay policy is arch-sensitive (CCCL uses
|
||
// default_delay_constructor_policy which picks fixed_delay for
|
||
// primitive types). We use fixed_delay as starting point.
|
||
// ============================================================
|
||
|
||
struct policy_selector {
|
||
|
||
constexpr BatchMemcpyPolicy operator()(const hardware_capability& hw) const {
|
||
// BI-V100: assume >= SM70 equivalent (no prefer_pow2_bits)
|
||
bool prefer_pow2 = false;
|
||
|
||
LargeBufferPolicy large{256, 32};
|
||
|
||
SmallBufferPolicy small{
|
||
/* threads_per_block = */ 128,
|
||
/* buffers_per_thread = */ 4,
|
||
/* bytes_per_thread = */ 8,
|
||
/* prefer_pow2_bits = */ prefer_pow2,
|
||
/* block_level_tile_size = */ large.threads_per_block * large.bytes_per_thread,
|
||
/* warp_level_threshold = */ 128,
|
||
/* block_level_threshold = */ 8 * 1024,
|
||
/* buffer_lookback_delay = */ {LookbackDelayAlgorithm::fixed_delay, 350, 450},
|
||
/* block_lookback_delay = */ {LookbackDelayAlgorithm::fixed_delay, 350, 450},
|
||
};
|
||
|
||
return {small, large};
|
||
}
|
||
};
|
||
|
||
} // namespace muh::tuning::batch_memcpy
|