EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix shared memory OOM on sm86 GPUs. (#4797)

Browse Source
This commit is contained in:
Yi Pan
2025-03-27 01:41:53 +08:00
committed by GitHub
parent d89c0e4b7e
commit 45fdf1f7f3
2 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
sgl-kernel/csrc/gemm/int8_gemm_kernel.cu
View File

@@ -703,8 +703,8 @@ torch::Tensor int8_scaled_mm(
sm75_dispatch_shape<cutlass::half_t, cutlass::arch::Sm75, cutlass::gemm::GemmShape<8, 8, 16>>(
out, mat_a, mat_b, scales_a, scales_b, bias);
} else if (sm_version >= 80 && sm_version < 90) {
// sm89 has a much smaller shared memory size (100K) than sm80 (160K)
if (sm_version == 89) {
// sm86/sm89 has a much smaller shared memory size (100K) than sm80 (160K)
if (sm_version == 86 || sm_version == 89) {
if (out_dtype == torch::kBFloat16) {
sm89_dispatch_shape<cutlass::bfloat16_t, cutlass::arch::Sm80, cutlass::gemm::GemmShape<16, 8, 32>>(
out, mat_a, mat_b, scales_a, scales_b, bias);