upstream: add GEMM kernel references from 4 repos for BI-V100 porting
Sources (all CUDA 10.2 compatible, no CUTLASS/Triton dependency): - leimao/CUDA-GEMM-Optimization: v00-v07, fp16 WMMA variant, double buffered - siboehm/SGEMM_CUDA: kernel 1-12, warp tiling + double buffering - wangzyon/NVIDIA_SGEMM_PRACTICE: kernel 1-7 - edtallison/sgemm-cuda: kernel 1-12 (reimplementation with notes) Key porting issue: ALL kernels hardcode WARPSIZE=32. BI-V100 has warp_size=64. Need to: 1. Replace all 32U / WARPSIZE constants with 64 2. Adjust warp subtile decomposition (WMITER, WNITER, WSUBM, WSUBN) 3. Adjust shared memory bank conflict avoidance (may have different bank count) 4. Test __shfl_down_sync with mask=0xFFFFFFFFFFFFFFFF (64-bit)
This commit is contained in:
Claude
19
upstream_ref/nvidia_sgemm_practice/kernel_1.cuh
Normal file
19
upstream_ref/nvidia_sgemm_practice/kernel_1.cuh
Normal file
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <cublas_v2.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
__global__ __launch_bounds__(1024) void
|
||||
mysgemm_v1(int M, int N, int K, float alpha, float *A, float *B, float beta, float *C) {
|
||||
|
||||
int gx = blockIdx.x * blockDim.x + threadIdx.x; // 全局x
|
||||
int gy = blockIdx.y * blockDim.y + threadIdx.y; // 全局y
|
||||
|
||||
float tmp = 0.;
|
||||
for (int i = 0; i < K; i++) {
|
||||
tmp += A[gy * K + i] * B[i * N + gx]; // 两次全局内存访问和一次FMA（累加乘）
|
||||
}
|
||||
C[gy * N + gx] = alpha * tmp + beta * C[gy * N + gx];
|
||||
}
|
||||
45
upstream_ref/nvidia_sgemm_practice/kernel_2.cuh
Normal file
45
upstream_ref/nvidia_sgemm_practice/kernel_2.cuh
Normal file
@@ -0,0 +1,45 @@
|
||||
#pragma once
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <cublas_v2.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
template<const int BLOCK_SIZE>
|
||||
__global__ void mysgemm_v2(int M, int N, int K, float alpha, float *A, float *B, float beta, float *C) {
|
||||
int bx = blockIdx.x;
|
||||
int by = blockIdx.y;
|
||||
|
||||
const int BM = BLOCK_SIZE;
|
||||
const int BN = BLOCK_SIZE;
|
||||
const int BK = BLOCK_SIZE;
|
||||
|
||||
int tx = threadIdx.x % BN;
|
||||
int ty = threadIdx.x / BN;
|
||||
|
||||
// 申请共享内存空间
|
||||
__shared__ float As[BM * BK];
|
||||
__shared__ float Bs[BK * BN];
|
||||
|
||||
// 移动到当前block
|
||||
A = &A[by * BM * K];
|
||||
B = &B[bx * BN];
|
||||
C = &C[by * BM * N + bx * BN];
|
||||
|
||||
float tmp = 0.;
|
||||
for (int k = 0; k < K; k += BK) {
|
||||
// 缓存A_tile和B_tile
|
||||
As[ty * BK + tx] = A[ty * K + tx];
|
||||
Bs[ty * BN + tx] = B[ty * N + tx];
|
||||
// 同步所有线程缓存完成
|
||||
__syncthreads();
|
||||
A += BK;
|
||||
B += BK * N;
|
||||
for (int i = 0; i < BK; i++) {
|
||||
tmp += As[ty * BK + i] * Bs[i * BN + tx];
|
||||
}
|
||||
// FMA计算需要读取缓存数据，在新一轮写入缓存前进行同步，确保所有线程计算完成
|
||||
__syncthreads();
|
||||
}
|
||||
C[ty * N + tx] = alpha * tmp + beta * C[ty * N + tx];
|
||||
}
|
||||
71
upstream_ref/nvidia_sgemm_practice/kernel_3.cuh
Normal file
71
upstream_ref/nvidia_sgemm_practice/kernel_3.cuh
Normal file
@@ -0,0 +1,71 @@
|
||||
#pragma once
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <cublas_v2.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
template<const int BM,
|
||||
const int BN,
|
||||
const int BK,
|
||||
const int TM>
|
||||
__global__ void mysgemm_v3(int M, int N, int K, float alpha, float *A, float *B, float beta, float *C) {
|
||||
int bx = blockIdx.x;
|
||||
int by = blockIdx.y;
|
||||
int thread_num = BM * BN / TM; // 一个线程负责block中计算TM个元素
|
||||
|
||||
int tx = threadIdx.x % BN;
|
||||
int ty = threadIdx.x / BN * TM;
|
||||
|
||||
__shared__ float As[BM * BK];
|
||||
__shared__ float Bs[BK * BN];
|
||||
|
||||
// 移动到当前block
|
||||
A = &A[by * BM * K];
|
||||
B = &B[bx * BN];
|
||||
C = &C[by * BM * N + bx * BN];
|
||||
|
||||
/*
|
||||
当前线程负责搬运全局内存中第a_tile_row行，第a_tile_col列元素至共享内存第a_tile_row行，第a_tile_col列
|
||||
a_tile_stride表示block中线程可搬运a_tile_stride行至共享内存；
|
||||
|
||||
若BM=64,BK=8,thread_num=512,则a_tile_stride=64,a_tile_stride=BM，表示每个线程搬运一轮即可完成所需元素的搬运;
|
||||
若BM=128,BK=8,thread_num=512,则a_tile_stride=64,表示每个线程搬运两轮即可完成所需元素的搬运;
|
||||
*/
|
||||
int a_tile_row = threadIdx.x / BK;
|
||||
int a_tile_col = threadIdx.x % BK;
|
||||
int a_tile_stride = thread_num / BK;
|
||||
|
||||
int b_tile_row = threadIdx.x / BN;
|
||||
int b_tile_col = threadIdx.x % BN;
|
||||
int b_tile_stride = thread_num / BN;
|
||||
|
||||
float tmp[TM + 1] = {0.}; // 每个线程负责TM个元素，则需要申请TM个寄存器保存累加值，额外的一个寄存器用于缓存；
|
||||
#pragma unroll
|
||||
for (int k = 0; k < K; k += BK) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BM; i += a_tile_stride) {
|
||||
As[(a_tile_row + i) * BK + a_tile_col] = A[(a_tile_row + i) * K + a_tile_col];
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BK; i += b_tile_stride) {
|
||||
Bs[(b_tile_row + i) * BN + b_tile_col] = B[(b_tile_row + i) * N + b_tile_col];
|
||||
}
|
||||
__syncthreads();
|
||||
A += BK;
|
||||
B += BK * N;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BK; i++) {
|
||||
tmp[TM] = Bs[tx + i * BN]; // 额外的一个寄存器，避免反复从共享内存中读取Bs[tx + i * BN]
|
||||
#pragma unroll // 循环展开，增加指令并行度
|
||||
for (int j = 0; j < TM; j++) {
|
||||
tmp[j] += As[(ty + j) * BK + i] * tmp[TM];
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < TM; j++) {
|
||||
C[(ty + j) * N + tx] = alpha * tmp[j] + beta * C[(ty + j) * N + tx];
|
||||
}
|
||||
}
|
||||
76
upstream_ref/nvidia_sgemm_practice/kernel_4.cuh
Normal file
76
upstream_ref/nvidia_sgemm_practice/kernel_4.cuh
Normal file
@@ -0,0 +1,76 @@
|
||||
#pragma once
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <cublas_v2.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
template<const int BM,
|
||||
const int BN,
|
||||
const int BK,
|
||||
const int TM,
|
||||
const int TN>
|
||||
__global__ void mysgemm_v4(int M, int N, int K, float alpha, float *A, float *B, float beta, float *C) {
|
||||
int bx = blockIdx.x;
|
||||
int by = blockIdx.y;
|
||||
|
||||
int block_row_thread = BN / TN;
|
||||
int block_col_thread = BM / TM;
|
||||
int thread_num = block_row_thread * block_col_thread; // 一个线程负责计算block中TM*TN个元素
|
||||
|
||||
int tx = (threadIdx.x % block_row_thread) * TN;
|
||||
int ty = (threadIdx.x / block_row_thread) * TM;
|
||||
|
||||
__shared__ float As[BM * BK];
|
||||
__shared__ float Bs[BK * BN];
|
||||
|
||||
// 移动到当前block
|
||||
A = &A[by * BM * K];
|
||||
B = &B[bx * BN];
|
||||
C = &C[by * BM * N + bx * BN];
|
||||
|
||||
/*
|
||||
当前线程负责搬运全局内存中第a_tile_row行，第a_tile_col列元素至共享内存第a_tile_row行，第a_tile_col列
|
||||
a_tile_stride表示block中线程可搬运a_tile_stride行至共享内存；
|
||||
|
||||
若BM=64,BK=8,thread_num=512,则a_tile_stride=64,a_tile_stride=BM，表示每个线程搬运一轮即可完成所需元素的搬运;
|
||||
若BM=128,BK=8,thread_num=512,则a_tile_stride=64,表示每个线程搬运两轮即可完成所需元素的搬运;
|
||||
*/
|
||||
int a_tile_row = threadIdx.x / BK;
|
||||
int a_tile_col = threadIdx.x % BK;
|
||||
int a_tile_stride = thread_num / BK;
|
||||
|
||||
int b_tile_row = threadIdx.x / BN;
|
||||
int b_tile_col = threadIdx.x % BN;
|
||||
int b_tile_stride = thread_num / BN;
|
||||
|
||||
float tmp[TM][TN] = {0.}; // 每个线程负责TM*TN个元素，则需要申请TM*TN个寄存器保存累加值，额外的一个寄存器用于缓存；
|
||||
#pragma unroll
|
||||
for (int k = 0; k < K; k += BK) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BM; i += a_tile_stride) {
|
||||
As[(a_tile_row + i) * BK + a_tile_col] = A[(a_tile_row + i) * K + a_tile_col];
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BK; i += b_tile_stride) {
|
||||
Bs[(b_tile_row + i) * BN + b_tile_col] = B[(b_tile_row + i) * N + b_tile_col];
|
||||
}
|
||||
__syncthreads();
|
||||
A += BK;
|
||||
B += BK * N;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BK; i++) {
|
||||
#pragma unroll // 循环展开，增加指令并行度
|
||||
for (int j = 0; j < TM; j++) {
|
||||
for (int l = 0; l < TN; l++)
|
||||
tmp[j][l] += As[(ty + j) * BK + i] * Bs[tx + l + i * BN];
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < TM; j++) {
|
||||
for (int l = 0; l < TN; l++)
|
||||
C[(ty + j) * N + tx + l] = alpha * tmp[j][l] + beta * C[(ty + j) * N + tx + l];
|
||||
}
|
||||
}
|
||||
88
upstream_ref/nvidia_sgemm_practice/kernel_5.cuh
Normal file
88
upstream_ref/nvidia_sgemm_practice/kernel_5.cuh
Normal file
@@ -0,0 +1,88 @@
|
||||
#pragma once
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <cublas_v2.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
template<const int BM,
|
||||
const int BN,
|
||||
const int BK,
|
||||
const int TM,
|
||||
const int TN>
|
||||
__global__ void mysgemm_v5(int M, int N, int K, float alpha, float *A, float *B, float beta, float *C) {
|
||||
int bx = blockIdx.x;
|
||||
int by = blockIdx.y;
|
||||
|
||||
int block_row_thread = BN / TN;
|
||||
int block_col_thread = BM / TM;
|
||||
int thread_num = block_row_thread * block_col_thread; // 一个线程负责计算block中TM*TN个元素
|
||||
|
||||
int tx = (threadIdx.x % block_row_thread) * TN;
|
||||
int ty = (threadIdx.x / block_row_thread) * TM;
|
||||
|
||||
__shared__ float As[BM * BK];
|
||||
__shared__ float Bs[BK * BN];
|
||||
|
||||
// 移动到当前block
|
||||
A = &A[by * BM * K];
|
||||
B = &B[bx * BN];
|
||||
C = &C[by * BM * N + bx * BN];
|
||||
|
||||
/*
|
||||
当前线程负责搬运全局内存中第a_tile_row行，第a_tile_col列元素至共享内存第a_tile_row行，第a_tile_col列
|
||||
a_tile_stride表示block中线程可搬运a_tile_stride行至共享内存；
|
||||
|
||||
若BM=64,BK=8,thread_num=512,则a_tile_stride=64,a_tile_stride=BM，表示每个线程搬运一轮即可完成所需元素的搬运;
|
||||
若BM=128,BK=8,thread_num=512,则a_tile_stride=64,表示每个线程搬运两轮即可完成所需元素的搬运;
|
||||
*/
|
||||
int a_tile_row = threadIdx.x / BK;
|
||||
int a_tile_col = threadIdx.x % BK;
|
||||
int a_tile_stride = thread_num / BK;
|
||||
|
||||
int b_tile_row = threadIdx.x / BN;
|
||||
int b_tile_col = threadIdx.x % BN;
|
||||
int b_tile_stride = thread_num / BN;
|
||||
|
||||
float tmp[TM][TN] = {0.}; // 每个线程负责TM*TN个元素，则需要申请TM*TN个寄存器保存累加值，额外的一个寄存器用于缓存；
|
||||
float a_frag[TM] = {0.};
|
||||
float b_frag[TN] = {0.};
|
||||
|
||||
#pragma unroll
|
||||
for (int k = 0; k < K; k += BK) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BM; i += a_tile_stride) {
|
||||
As[(a_tile_row + i) * BK + a_tile_col] = A[(a_tile_row + i) * K + a_tile_col];
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BK; i += b_tile_stride) {
|
||||
Bs[(b_tile_row + i) * BN + b_tile_col] = B[(b_tile_row + i) * N + b_tile_col];
|
||||
}
|
||||
__syncthreads();
|
||||
A += BK;
|
||||
B += BK * N;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BK; i++) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < TM; j++) {
|
||||
a_frag[j] = As[(ty + j) * BK + i];
|
||||
}
|
||||
#pragma unroll
|
||||
for (int l = 0; l < TN; l++) {
|
||||
b_frag[l] = Bs[tx + l + i * BN];
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < TM; j++) {
|
||||
#pragma unroll
|
||||
for (int l = 0; l < TN; l++)
|
||||
tmp[j][l] += a_frag[j] * b_frag[l];
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < TM; j++) {
|
||||
for (int l = 0; l < TN; l++)
|
||||
C[(ty + j) * N + tx + l] = alpha * tmp[j][l] + beta * C[(ty + j) * N + tx + l];
|
||||
}
|
||||
}
|
||||
110
upstream_ref/nvidia_sgemm_practice/kernel_6.cuh
Normal file
110
upstream_ref/nvidia_sgemm_practice/kernel_6.cuh
Normal file
@@ -0,0 +1,110 @@
|
||||
#pragma once
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <cublas_v2.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#define OFFSET(row, col, ld) ((row)*(ld)+(col))
|
||||
#define FETCH_FLOAT4(pointer) (reinterpret_cast<float4*>(&(pointer))[0])
|
||||
|
||||
template<const int BM,
|
||||
const int BN,
|
||||
const int BK,
|
||||
const int TM,
|
||||
const int TN>
|
||||
__global__ void mysgemm_v6(int M, int N, int K, float alpha, float *A, float *B, float beta, float *C) {
|
||||
int bx = blockIdx.x;
|
||||
int by = blockIdx.y;
|
||||
|
||||
const int block_row_thread = BN / TN;
|
||||
const int block_col_thread = BM / TM;
|
||||
const int thread_num = block_row_thread * block_col_thread; // 一个线程负责计算block中TM*TN个元素
|
||||
|
||||
// 当前线程对应thread tile的左上角元素在block中的位置
|
||||
int tx = (threadIdx.x % block_row_thread) * TN;
|
||||
int ty = (threadIdx.x / block_row_thread) * TM;
|
||||
|
||||
__shared__ float As[BK * BM];
|
||||
__shared__ float Bs[BK * BN];
|
||||
|
||||
|
||||
const int ldg_a_num = BK * BM / thread_num / 4; // 每个线程搬运4个浮点数，完成搬运至As需要所有线程搬运ldg_a_num轮
|
||||
const int ldg_b_num = BK * BN / thread_num / 4; // 每个线程搬运4个浮点数，完成搬运至Bs需要所有线程搬运ldg_b_num轮
|
||||
|
||||
int a_tile_row = threadIdx.x / (BK / 4); // 每行4个字节作为一个内存块，当前线程负责第a_tile_row行的第a_tile_col个内存块的搬运
|
||||
int a_tile_col = threadIdx.x % (BK / 4) * 4;
|
||||
int a_tile_stride = BM / ldg_a_num; // 一共BM行，搬运ldg_a_num轮，每论搬运a_tile_stride行
|
||||
|
||||
int b_tile_row = threadIdx.x / (BN / 4); // 每行4个字节作为一个内存块，当前线程负责第b_tile_row行的第b_tile_col个内存块的搬运
|
||||
int b_tile_col = threadIdx.x % (BN / 4) * 4;
|
||||
int b_tile_stride = BK / ldg_b_num; // 一共BK行，搬运ldg_b_num轮，每论搬运b_tile_stride行
|
||||
|
||||
float accum[TM][TN] = {0.}; // 每个线程负责TM*TN个元素，则需要申请TM*TN个寄存器保存累加值，额外的一个寄存器用于缓存；
|
||||
|
||||
// 计算ldg_a_num的所有参数必须全部是const，否则不能用来申明数组大小
|
||||
float ldg_a_reg[4 * ldg_a_num] = {0.}; // 每个线程搬运ldg_a_num轮，寄存器缓存ldg_a_num个float4元素，用于转置As矩阵
|
||||
|
||||
float a_frag[TM]; // 缓存As共享内存
|
||||
float b_frag[TN]; // 缓存Bs共享内存
|
||||
|
||||
// 移动到当前block
|
||||
A = &A[by * BM * K];
|
||||
B = &B[bx * BN];
|
||||
C = &C[by * BM * N + bx * BN];
|
||||
|
||||
#pragma unroll
|
||||
for (int k = 0; k < K; k += BK) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BM; i += a_tile_stride) {
|
||||
int ldg_index = i / a_tile_stride * 4; // 第ldg_index轮
|
||||
FETCH_FLOAT4(ldg_a_reg[ldg_index]) =
|
||||
FETCH_FLOAT4(A[OFFSET(a_tile_row + i, a_tile_col, K)]);
|
||||
// As转置存，其中ldg_a_reg做中间缓存，目的是读取时可以按FLOAT4读取
|
||||
As[OFFSET(a_tile_col, i + a_tile_row, BM)] = ldg_a_reg[ldg_index];
|
||||
As[OFFSET(a_tile_col + 1, i + a_tile_row, BM)] = ldg_a_reg[ldg_index + 1];
|
||||
As[OFFSET(a_tile_col + 2, i + a_tile_row, BM)] = ldg_a_reg[ldg_index + 2];
|
||||
As[OFFSET(a_tile_col + 3, i + a_tile_row, BM)] = ldg_a_reg[ldg_index + 3];
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BK; i += b_tile_stride) {
|
||||
FETCH_FLOAT4(Bs[OFFSET(b_tile_row + i, b_tile_col, BN)]) =
|
||||
FETCH_FLOAT4(B[OFFSET(b_tile_row + i, b_tile_col, N)]); // 不需要转置
|
||||
}
|
||||
__syncthreads();
|
||||
A += BK;
|
||||
B += BK * N;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BK; i++) {
|
||||
#pragma unroll
|
||||
for (int m = 0; m < TM; m += 4) {
|
||||
FETCH_FLOAT4(a_frag[m]) = FETCH_FLOAT4(As[OFFSET(i, ty + m, BM)]); // 偏移到当前thread tile
|
||||
}
|
||||
#pragma unroll
|
||||
for (int n = 0; n < TN; n += 4) {
|
||||
FETCH_FLOAT4(b_frag[n]) = FETCH_FLOAT4(Bs[OFFSET(i, tx + n, BN)]); // 偏移到当前thread tile
|
||||
}
|
||||
#pragma unroll
|
||||
for (int m = 0; m < TM; m++) {
|
||||
#pragma unroll
|
||||
for (int n = 0; n < TN; n++) {
|
||||
accum[m][n] += a_frag[m] * b_frag[n];
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
#pragma unroll
|
||||
for (int m = 0; m < TM; m++) {
|
||||
#pragma unroll
|
||||
for (int n = 0; n < TN; n += 4) {
|
||||
float4 ctmp = FETCH_FLOAT4(C[OFFSET(ty + m, tx + n, N)]);
|
||||
//float4 atmp = FETCH_FLOAT4(accum[m][n]);
|
||||
ctmp.x = alpha * accum[m][n] + beta * ctmp.x;
|
||||
ctmp.y = alpha * accum[m][n + 1] + beta * ctmp.y;
|
||||
ctmp.z = alpha * accum[m][n + 2] + beta * ctmp.z;
|
||||
ctmp.w = alpha * accum[m][n + 3] + beta * ctmp.w;
|
||||
FETCH_FLOAT4(C[OFFSET(ty + m, tx + n, N)]) = ctmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
180
upstream_ref/nvidia_sgemm_practice/kernel_7.cuh
Normal file
180
upstream_ref/nvidia_sgemm_practice/kernel_7.cuh
Normal file
@@ -0,0 +1,180 @@
|
||||
#pragma once
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <cublas_v2.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#define OFFSET(row, col, ld) ((row)*(ld)+(col))
|
||||
#define FETCH_FLOAT4(pointer) (reinterpret_cast<float4*>(&(pointer))[0])
|
||||
|
||||
template<const int BM,
|
||||
const int BN,
|
||||
const int BK,
|
||||
const int TM,
|
||||
const int TN>
|
||||
__global__ void mysgemm_v7(int M, int N, int K, float alpha, float *A, float *B, float beta, float *C) {
|
||||
int bx = blockIdx.x;
|
||||
int by = blockIdx.y;
|
||||
|
||||
const int block_row_thread = BN / TN;
|
||||
const int block_col_thread = BM / TM;
|
||||
const int thread_num = block_row_thread * block_col_thread; // 一个线程负责计算block中TM*TN个元素
|
||||
|
||||
// 当前线程对应thread tile的左上角元素在block中的位置
|
||||
int tx = (threadIdx.x % block_row_thread) * TN;
|
||||
int ty = (threadIdx.x / block_row_thread) * TM;
|
||||
|
||||
__shared__ float As[2][BK * BM]; // 增加一倍共享内存大小用于缓存
|
||||
__shared__ float Bs[2][BK * BN];
|
||||
|
||||
|
||||
const int ldg_a_num = BK * BM / thread_num / 4; // 每个线程搬运4个浮点数，完成搬运至As需要所有线程搬运ldg_a_num轮
|
||||
const int ldg_b_num = BK * BN / thread_num / 4; // 每个线程搬运4个浮点数，完成搬运至Bs需要所有线程搬运ldg_b_num轮
|
||||
|
||||
int a_tile_row = threadIdx.x / (BK / 4); // 每行4个字节作为一个内存块，当前线程负责第a_tile_row行的第a_tile_col个内存块的搬运
|
||||
int a_tile_col = threadIdx.x % (BK / 4) * 4;
|
||||
int a_tile_stride = BM / ldg_a_num; // 一共BM行，搬运ldg_a_num轮，每论搬运a_tile_stride行
|
||||
|
||||
int b_tile_row = threadIdx.x / (BN / 4); // 每行4个字节作为一个内存块，当前线程负责第b_tile_row行的第b_tile_col个内存块的搬运
|
||||
int b_tile_col = threadIdx.x % (BN / 4) * 4;
|
||||
int b_tile_stride = BK / ldg_b_num; // 一共BK行，搬运ldg_b_num轮，每论搬运b_tile_stride行
|
||||
|
||||
float accum[TM][TN] = {0.}; // 每个线程负责TM*TN个元素，则需要申请TM*TN个寄存器保存累加值，额外的一个寄存器用于缓存；
|
||||
|
||||
// 计算ldg_a_num的所有参数必须全部是const，否则不能用来申明数组大小
|
||||
float ldg_a_reg[4 * ldg_a_num] = {0.}; // 每个线程搬运ldg_a_num轮，寄存器缓存ldg_a_num个float4元素，用于转置As矩阵
|
||||
float ldg_b_reg[4 * ldg_b_num] = {0.}; // 每个线程搬运ldg_a_num轮，寄存器缓存ldg_a_num个float4元素，用于转置As矩阵
|
||||
|
||||
float a_frag[2][TM]; // 缓存As共享内存,增加一倍寄存器大小用于缓存
|
||||
float b_frag[2][TN]; // 缓存Bs共享内存,增加一倍寄存器大小用于缓存
|
||||
|
||||
// 移动到当前block
|
||||
A = &A[by * BM * K];
|
||||
B = &B[bx * BN];
|
||||
C = &C[by * BM * N + bx * BN];
|
||||
|
||||
// first global to shared
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BM; i += a_tile_stride) {
|
||||
int ldg_index = i / a_tile_stride * 4; // 第ldg_index轮
|
||||
FETCH_FLOAT4(ldg_a_reg[ldg_index]) =
|
||||
FETCH_FLOAT4(A[OFFSET(a_tile_row + i, a_tile_col, K)]);
|
||||
// As转置存，其中ldg_a_reg做中间缓存，目的是读取时可以按FLOAT4读取
|
||||
As[0][OFFSET(a_tile_col, i + a_tile_row, BM)] = ldg_a_reg[ldg_index];
|
||||
As[0][OFFSET(a_tile_col + 1, i + a_tile_row, BM)] = ldg_a_reg[ldg_index + 1];
|
||||
As[0][OFFSET(a_tile_col + 2, i + a_tile_row, BM)] = ldg_a_reg[ldg_index + 2];
|
||||
As[0][OFFSET(a_tile_col + 3, i + a_tile_row, BM)] = ldg_a_reg[ldg_index + 3];
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BK; i += b_tile_stride) {
|
||||
FETCH_FLOAT4(Bs[0][OFFSET(b_tile_row + i, b_tile_col, BN)]) =
|
||||
FETCH_FLOAT4(B[OFFSET(b_tile_row + i, b_tile_col, N)]); // 不需要转置
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// first shared to frag
|
||||
#pragma unroll
|
||||
for (int m = 0; m < TM; m += 4) {
|
||||
FETCH_FLOAT4(a_frag[0][m]) = FETCH_FLOAT4(As[0][OFFSET(0, ty + m, BM)]); // 偏移到当前thread tile
|
||||
}
|
||||
#pragma unroll
|
||||
for (int n = 0; n < TN; n += 4) {
|
||||
FETCH_FLOAT4(b_frag[0][n]) = FETCH_FLOAT4(Bs[0][OFFSET(0, tx + n, BN)]); // 偏移到当前thread tile
|
||||
}
|
||||
|
||||
|
||||
int write_index = 1;
|
||||
int load_index;
|
||||
int k = 0;
|
||||
do {
|
||||
k += BK;
|
||||
// load global to reg
|
||||
if (k < K) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BM; i += a_tile_stride) {
|
||||
int ldg_index = i / a_tile_stride * 4; // 第ldg_index轮
|
||||
FETCH_FLOAT4(ldg_a_reg[ldg_index]) =
|
||||
FETCH_FLOAT4(A[OFFSET(a_tile_row + i, k + a_tile_col, K)]);
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BK; i += b_tile_stride) {
|
||||
int ldg_index = i / b_tile_stride * 4; // 第ldg_index轮
|
||||
FETCH_FLOAT4(ldg_b_reg[ldg_index]) =
|
||||
FETCH_FLOAT4(B[OFFSET(k + b_tile_row + i, b_tile_col, N)]);
|
||||
}
|
||||
}
|
||||
|
||||
load_index = write_index ^ 1;
|
||||
#pragma unroll
|
||||
for (int bk = 0; bk < BK - 1; bk++) {
|
||||
for (int m = 0; m < TM; m += 4) {
|
||||
FETCH_FLOAT4(a_frag[(bk + 1) % 2][m]) = FETCH_FLOAT4(
|
||||
As[load_index][OFFSET(bk + 1, ty + m, BM)]); // 偏移到当前thread tile
|
||||
}
|
||||
#pragma unroll
|
||||
for (int n = 0; n < TN; n += 4) {
|
||||
FETCH_FLOAT4(b_frag[(bk + 1) % 2][n]) = FETCH_FLOAT4(
|
||||
Bs[load_index][OFFSET(bk + 1, tx + n, BN)]); // 偏移到当前thread tile
|
||||
}
|
||||
#pragma unroll
|
||||
for (int m = 0; m < TM; m++) {
|
||||
for (int n = 0; n < TN; n++) {
|
||||
accum[m][n] += a_frag[bk % 2][m] * b_frag[bk % 2][n];
|
||||
}
|
||||
}
|
||||
}
|
||||
if (k < K) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BM; i += a_tile_stride) {
|
||||
int ldg_index = i / a_tile_stride * 4;
|
||||
As[write_index][OFFSET(a_tile_col, i + a_tile_row, BM)] = ldg_a_reg[ldg_index];
|
||||
As[write_index][OFFSET(a_tile_col + 1, i + a_tile_row, BM)] = ldg_a_reg[ldg_index + 1];
|
||||
As[write_index][OFFSET(a_tile_col + 2, i + a_tile_row, BM)] = ldg_a_reg[ldg_index + 2];
|
||||
As[write_index][OFFSET(a_tile_col + 3, i + a_tile_row, BM)] = ldg_a_reg[ldg_index + 3];
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < BK; i += b_tile_stride) {
|
||||
int ldg_index = i / b_tile_stride * 4;
|
||||
FETCH_FLOAT4(Bs[write_index][OFFSET(b_tile_row + i, b_tile_col, BN)]) =
|
||||
FETCH_FLOAT4(ldg_b_reg[ldg_index]);
|
||||
}
|
||||
__syncthreads();
|
||||
#pragma unroll
|
||||
for (int m = 0; m < TM; m += 4) {
|
||||
FETCH_FLOAT4(a_frag[0][m]) = FETCH_FLOAT4(
|
||||
As[write_index][OFFSET(0, ty + m, BM)]); // 偏移到当前thread tile
|
||||
}
|
||||
#pragma unroll
|
||||
for (int n = 0; n < TN; n += 4) {
|
||||
FETCH_FLOAT4(b_frag[0][n]) = FETCH_FLOAT4(
|
||||
Bs[write_index][OFFSET(0, tx + n, BN)]); // 偏移到当前thread tile
|
||||
}
|
||||
|
||||
write_index ^= 1;
|
||||
}
|
||||
#pragma unroll
|
||||
for (int m = 0; m < TM; m++) {
|
||||
#pragma unroll
|
||||
for (int n = 0; n < TN; n++) {
|
||||
accum[m][n] += a_frag[(BK - 1) % 2][m] * b_frag[(BK - 1) % 2][n];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} while (k < K);
|
||||
|
||||
// C = alpha*AB+C
|
||||
#pragma unroll
|
||||
for (int m = 0; m < TM; m++) {
|
||||
#pragma unroll
|
||||
for (int n = 0; n < TN; n += 4) {
|
||||
float4 ctmp = FETCH_FLOAT4(C[OFFSET(ty + m, tx + n, N)]);
|
||||
ctmp.x = alpha * accum[m][n] + beta * ctmp.x;
|
||||
ctmp.y = alpha * accum[m][n + 1] + beta * ctmp.y;
|
||||
ctmp.z = alpha * accum[m][n + 2] + beta * ctmp.z;
|
||||
ctmp.w = alpha * accum[m][n + 3] + beta * ctmp.w;
|
||||
FETCH_FLOAT4(C[OFFSET(ty + m, tx + n, N)]) = ctmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
119
upstream_ref/nvidia_sgemm_practice/sgemm.cu
Normal file
119
upstream_ref/nvidia_sgemm_practice/sgemm.cu
Normal file
@@ -0,0 +1,119 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/time.h>
|
||||
#include <utils.cuh>
|
||||
|
||||
#define cudaCheck(err) (cudaCheck(err, __FILE__, __LINE__))
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc != 2) {
|
||||
printf("Please select a kernel (range 0 - 11, here 0 is for NVIDIA cuBLAS).\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
// cuda kernel num
|
||||
int kernel_num = atoi(argv[1]);
|
||||
if (kernel_num < 0 || kernel_num > 11) {
|
||||
printf("Please enter a valid kernel number (0-11).\n");
|
||||
exit(EXIT_FAILURE);
|
||||
} else {
|
||||
printf("Select kernel %d.\n", kernel_num);
|
||||
};
|
||||
|
||||
// 申明句柄，创建句柄, cublasCreate会返回一个cublasStatus_t类型的值，用来判断句柄是否创建成功(值为0)
|
||||
cublasHandle_t handle;
|
||||
if (cublasCreate(&handle)) {
|
||||
printf("Create cublas handle error.\n");
|
||||
exit(EXIT_FAILURE);
|
||||
};
|
||||
|
||||
// 采用cudaEvent进行gpu流计时，cudaEvent相当于在目标流中发布事件任务
|
||||
float elapsed_time;
|
||||
cudaEvent_t beg, end;
|
||||
cudaEventCreate(&beg);
|
||||
cudaEventCreate(&end);
|
||||
|
||||
// matrix size
|
||||
int size_len = 24;
|
||||
int SIZE[size_len];
|
||||
for (int i = 0; i < size_len; i++)
|
||||
SIZE[i] = 256 * (i + 1);
|
||||
|
||||
int m, n, k, max_size;
|
||||
max_size = SIZE[size_len - 1];
|
||||
printf("max_size=%d\n", max_size);
|
||||
|
||||
float alpha = 1.0, beta = 0.; //two arbitary input parameters，C=α*AB+β*C
|
||||
|
||||
float *A = NULL, *B = NULL, *C = NULL, *C_ref = NULL; //host matrices
|
||||
float *dA = NULL, *dB = NULL, *dC = NULL, *dC_ref = NULL; //device matrices
|
||||
|
||||
A = (float *) malloc(sizeof(float) * max_size * max_size);
|
||||
B = (float *) malloc(sizeof(float) * max_size * max_size);
|
||||
C = (float *) malloc(sizeof(float) * max_size * max_size);
|
||||
C_ref = (float *) malloc(sizeof(float) * max_size * max_size);
|
||||
|
||||
randomize_matrix(A, max_size * max_size);
|
||||
randomize_matrix(B, max_size * max_size);
|
||||
randomize_matrix(C, max_size * max_size);
|
||||
copy_matrix(C, C_ref, max_size * max_size);
|
||||
|
||||
cudaCheck(cudaMalloc((void **) &dA, sizeof(float) * max_size * max_size));
|
||||
cudaCheck(cudaMalloc((void **) &dB, sizeof(float) * max_size * max_size));
|
||||
cudaCheck(cudaMalloc((void **) &dC, sizeof(float) * max_size * max_size));
|
||||
cudaCheck(cudaMalloc((void **) &dC_ref, sizeof(float) * max_size * max_size));
|
||||
|
||||
cudaCheck(cudaMemcpy(dA, A, sizeof(float) * max_size * max_size, cudaMemcpyHostToDevice));
|
||||
cudaCheck(cudaMemcpy(dB, B, sizeof(float) * max_size * max_size, cudaMemcpyHostToDevice));
|
||||
cudaCheck(cudaMemcpy(dC, C, sizeof(float) * max_size * max_size, cudaMemcpyHostToDevice));
|
||||
cudaCheck(cudaMemcpy(dC_ref, C_ref, sizeof(float) * max_size * max_size, cudaMemcpyHostToDevice));
|
||||
|
||||
int repeat_times = 10;
|
||||
for (int i = 0; i < size_len; i++) {
|
||||
m = n = k = SIZE[i];
|
||||
|
||||
printf("m=n=k=%d\n", m);
|
||||
// 验证计算正确性，同时在核函数计时前预先执行一次，避免冷启动误差
|
||||
if (kernel_num != 0) {
|
||||
test_kernel(0, m, n, k, alpha, dA, dB, beta, dC_ref, handle); // cuBLAS
|
||||
test_kernel(kernel_num, m, n, k, alpha, dA, dB, beta, dC, handle); // user define
|
||||
cudaDeviceSynchronize();
|
||||
cudaMemcpy(C, dC, sizeof(float) * m * n, cudaMemcpyDeviceToHost);
|
||||
cudaMemcpy(C_ref, dC_ref, sizeof(float) * m * n, cudaMemcpyDeviceToHost);
|
||||
cudaDeviceSynchronize();
|
||||
|
||||
if (!verify_matrix(C_ref, C, m * n)) {
|
||||
printf("Failed to pass the correctness verification against NVIDIA cuBLAS. Exited.\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
cudaDeviceSynchronize();
|
||||
|
||||
cudaEventRecord(beg);
|
||||
for (int j = 0; j < repeat_times; j++) {
|
||||
test_kernel(kernel_num, m, n, k, alpha, dA, dB, beta, dC, handle);
|
||||
}
|
||||
cudaEventRecord(end);
|
||||
cudaEventSynchronize(beg);
|
||||
cudaEventSynchronize(end);
|
||||
cudaEventElapsedTime(&elapsed_time, beg, end);
|
||||
elapsed_time /= 1000.; //换算成秒
|
||||
|
||||
printf("Average elasped time: (%f) second, performance: (%f) GFLOPS. size: (%d).\n",
|
||||
elapsed_time / repeat_times, 2. * 1e-9 * repeat_times * m * n * k / elapsed_time, m);
|
||||
fflush(stdout);
|
||||
copy_matrix(C_ref, C, m * n); //sync C with cuBLAS to prepare for the next run
|
||||
}
|
||||
|
||||
// 释放CPU和GPU空间
|
||||
free(A);
|
||||
free(B);
|
||||
free(C);
|
||||
free(C_ref);
|
||||
cudaFree(dA);
|
||||
cudaFree(dB);
|
||||
cudaFree(dC);
|
||||
cudaFree(dC_ref);
|
||||
|
||||
return 0;
|
||||
};
|
||||
42
upstream_ref/nvidia_sgemm_practice/utils.cuh
Normal file
42
upstream_ref/nvidia_sgemm_practice/utils.cuh
Normal file
@@ -0,0 +1,42 @@
|
||||
#pragma once
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/time.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <cublas_v2.h>
|
||||
|
||||
/*
|
||||
=====================================
|
||||
CUDA操作
|
||||
=====================================
|
||||
*/
|
||||
void cudaCheck(cudaError_t error, const char *file, int line); //CUDA错误检查
|
||||
void CudaDeviceInfo(); // 打印CUDA信息
|
||||
|
||||
/*
|
||||
=====================================
|
||||
矩阵操作
|
||||
=====================================
|
||||
*/
|
||||
void randomize_matrix(float *mat, int N); // 随机初始化矩阵
|
||||
void copy_matrix(float *src, float *dest, int N); // 复制矩阵
|
||||
void print_matrix(const float *A, int M, int N); // 打印矩阵
|
||||
bool verify_matrix(float *mat1, float *mat2, int N); // 验证矩阵
|
||||
|
||||
/*
|
||||
=====================================
|
||||
计时操作
|
||||
=====================================
|
||||
*/
|
||||
float get_current_sec(); // 获取当前时刻
|
||||
float cpu_elapsed_time(float &beg, float &end); // 计算时间差
|
||||
|
||||
/*
|
||||
=====================================
|
||||
kernel操作
|
||||
=====================================
|
||||
*/
|
||||
//调用指定核函数计算矩阵乘法
|
||||
void test_kernel(int kernel_num, int m, int n, int k, float alpha, float *A, float *B, float beta, float *C, cublasHandle_t handle);
|
||||
Reference in New Issue
Block a user