9ca33cf4d5
Sources (all CUDA 10.2 compatible, no CUTLASS/Triton dependency): - leimao/CUDA-GEMM-Optimization: v00-v07, fp16 WMMA variant, double buffered - siboehm/SGEMM_CUDA: kernel 1-12, warp tiling + double buffering - wangzyon/NVIDIA_SGEMM_PRACTICE: kernel 1-7 - edtallison/sgemm-cuda: kernel 1-12 (reimplementation with notes) Key porting issue: ALL kernels hardcode WARPSIZE=32. BI-V100 has warp_size=64. Need to: 1. Replace all 32U / WARPSIZE constants with 64 2. Adjust warp subtile decomposition (WMITER, WNITER, WSUBM, WSUBN) 3. Adjust shared memory bank conflict avoidance (may have different bank count) 4. Test __shfl_down_sync with mask=0xFFFFFFFFFFFFFFFF (64-bit)
88 lines
3.0 KiB
Plaintext
88 lines
3.0 KiB
Plaintext
#pragma once
|
||
|
||
#include <cuda_runtime.h>
|
||
#include <cublas_v2.h>
|
||
#include <stdio.h>
|
||
#include <stdlib.h>
|
||
|
||
template<const int BM,
|
||
const int BN,
|
||
const int BK,
|
||
const int TM,
|
||
const int TN>
|
||
__global__ void mysgemm_v5(int M, int N, int K, float alpha, float *A, float *B, float beta, float *C) {
|
||
int bx = blockIdx.x;
|
||
int by = blockIdx.y;
|
||
|
||
int block_row_thread = BN / TN;
|
||
int block_col_thread = BM / TM;
|
||
int thread_num = block_row_thread * block_col_thread; // 一个线程负责计算block中TM*TN个元素
|
||
|
||
int tx = (threadIdx.x % block_row_thread) * TN;
|
||
int ty = (threadIdx.x / block_row_thread) * TM;
|
||
|
||
__shared__ float As[BM * BK];
|
||
__shared__ float Bs[BK * BN];
|
||
|
||
// 移动到当前block
|
||
A = &A[by * BM * K];
|
||
B = &B[bx * BN];
|
||
C = &C[by * BM * N + bx * BN];
|
||
|
||
/*
|
||
当前线程负责搬运全局内存中第a_tile_row行，第a_tile_col列元素至共享内存第a_tile_row行，第a_tile_col列
|
||
a_tile_stride表示block中线程可搬运a_tile_stride行至共享内存；
|
||
|
||
若BM=64,BK=8,thread_num=512,则a_tile_stride=64,a_tile_stride=BM，表示每个线程搬运一轮即可完成所需元素的搬运;
|
||
若BM=128,BK=8,thread_num=512,则a_tile_stride=64,表示每个线程搬运两轮即可完成所需元素的搬运;
|
||
*/
|
||
int a_tile_row = threadIdx.x / BK;
|
||
int a_tile_col = threadIdx.x % BK;
|
||
int a_tile_stride = thread_num / BK;
|
||
|
||
int b_tile_row = threadIdx.x / BN;
|
||
int b_tile_col = threadIdx.x % BN;
|
||
int b_tile_stride = thread_num / BN;
|
||
|
||
float tmp[TM][TN] = {0.}; // 每个线程负责TM*TN个元素，则需要申请TM*TN个寄存器保存累加值，额外的一个寄存器用于缓存；
|
||
float a_frag[TM] = {0.};
|
||
float b_frag[TN] = {0.};
|
||
|
||
#pragma unroll
|
||
for (int k = 0; k < K; k += BK) {
|
||
#pragma unroll
|
||
for (int i = 0; i < BM; i += a_tile_stride) {
|
||
As[(a_tile_row + i) * BK + a_tile_col] = A[(a_tile_row + i) * K + a_tile_col];
|
||
}
|
||
#pragma unroll
|
||
for (int i = 0; i < BK; i += b_tile_stride) {
|
||
Bs[(b_tile_row + i) * BN + b_tile_col] = B[(b_tile_row + i) * N + b_tile_col];
|
||
}
|
||
__syncthreads();
|
||
A += BK;
|
||
B += BK * N;
|
||
#pragma unroll
|
||
for (int i = 0; i < BK; i++) {
|
||
#pragma unroll
|
||
for (int j = 0; j < TM; j++) {
|
||
a_frag[j] = As[(ty + j) * BK + i];
|
||
}
|
||
#pragma unroll
|
||
for (int l = 0; l < TN; l++) {
|
||
b_frag[l] = Bs[tx + l + i * BN];
|
||
}
|
||
#pragma unroll
|
||
for (int j = 0; j < TM; j++) {
|
||
#pragma unroll
|
||
for (int l = 0; l < TN; l++)
|
||
tmp[j][l] += a_frag[j] * b_frag[l];
|
||
}
|
||
}
|
||
__syncthreads();
|
||
}
|
||
#pragma unroll
|
||
for (int j = 0; j < TM; j++) {
|
||
for (int l = 0; l < TN; l++)
|
||
C[(ty + j) * N + tx + l] = alpha * tmp[j][l] + beta * C[(ty + j) * N + tx + l];
|
||
}
|
||
} |