[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
EngineX CI
284
cccl_upstream/libcudacxx/test/simd_codegen/CMakeLists.txt
Normal file
284
cccl_upstream/libcudacxx/test/simd_codegen/CMakeLists.txt
Normal file
@@ -0,0 +1,284 @@
|
||||
##===----------------------------------------------------------------------===##
|
||||
##
|
||||
## Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
## under the Apache License v2.0 with LLVM Exceptions.
|
||||
## See https://llvm.org/LICENSE.txt for license information.
|
||||
## SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
## SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
##
|
||||
##===----------------------------------------------------------------------===##
|
||||
|
||||
add_custom_target(libcudacxx.test.simd.ptx)
|
||||
add_custom_target(libcudacxx.test.simd.sass)
|
||||
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
# SETUP: skip unsupported compilers, find tools, set up CUDA architectures
|
||||
|
||||
# GCC7 and NVCC 12.0 do not vectorize load/store.
|
||||
if (
|
||||
CMAKE_CXX_COMPILER_ID STREQUAL GNU
|
||||
AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 7
|
||||
AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8
|
||||
)
|
||||
message("-- Skipping simd codegen tests with GCC 7 host compiler")
|
||||
return()
|
||||
endif()
|
||||
|
||||
if (
|
||||
CMAKE_CUDA_COMPILER_ID STREQUAL NVIDIA
|
||||
AND CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0
|
||||
AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 12.1
|
||||
)
|
||||
message("-- Skipping simd codegen tests with NVCC 12.0")
|
||||
return()
|
||||
endif()
|
||||
|
||||
# skip nvc++
|
||||
if (CMAKE_CXX_COMPILER_ID STREQUAL NVHPC)
|
||||
message("-- Skipping simd codegen tests with NVHPC host compiler")
|
||||
return()
|
||||
endif()
|
||||
|
||||
find_program(
|
||||
filecheck
|
||||
NAMES FileCheck FileCheck-21 FileCheck-20 FileCheck-19 FileCheck-18
|
||||
)
|
||||
|
||||
if (filecheck)
|
||||
message("-- ${filecheck} found... building simd codegen tests")
|
||||
else()
|
||||
return()
|
||||
endif()
|
||||
|
||||
if ("Clang" STREQUAL "${CMAKE_CUDA_COMPILER_ID}")
|
||||
message("-- clang-cuda does not support the -dc simd codegen tests")
|
||||
return()
|
||||
endif()
|
||||
|
||||
find_program(cuobjdump "cuobjdump" REQUIRED)
|
||||
find_program(bash "bash" REQUIRED)
|
||||
|
||||
set(
|
||||
dump_and_check_script
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/../atomic_codegen/dump_and_check.bash"
|
||||
)
|
||||
|
||||
set(simd_codegen_sass_cuda_archs 80 90)
|
||||
if (
|
||||
CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.8
|
||||
AND NOT CMAKE_CUDA_COMPILER_ID STREQUAL Clang
|
||||
)
|
||||
list(APPEND simd_codegen_sass_cuda_archs 100 120)
|
||||
endif()
|
||||
|
||||
if (
|
||||
CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.9
|
||||
AND NOT CMAKE_CUDA_COMPILER_ID STREQUAL Clang
|
||||
)
|
||||
list(APPEND simd_codegen_sass_cuda_archs 103 120f)
|
||||
endif()
|
||||
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
# HELPER FUNCTIONS
|
||||
|
||||
function(simd_codegen_set_cuda_arch target_name arch)
|
||||
if (arch MATCHES "[af]$")
|
||||
set_target_properties(${target_name} PROPERTIES CUDA_ARCHITECTURES OFF)
|
||||
target_compile_options(
|
||||
${target_name}
|
||||
PRIVATE "--generate-code=arch=compute_${arch},code=sm_${arch}"
|
||||
)
|
||||
else()
|
||||
set_target_properties(
|
||||
${target_name}
|
||||
PROPERTIES CUDA_ARCHITECTURES "${arch}"
|
||||
)
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
# Given the test file content, extract the SM archs that are checked for.
|
||||
# SMXX: any arch
|
||||
# SM1XX: any SM100-series arch
|
||||
# SM100-PLUS: SM100 and newer
|
||||
function(simd_codegen_get_sass_check_prefixes out_var test_contents arch)
|
||||
set(check_prefixes SMXX)
|
||||
set(arch_prefix "SM${arch}")
|
||||
|
||||
string(
|
||||
REGEX MATCH
|
||||
"; ${arch_prefix}(:|-LABEL:|-NOT:|-NEXT:|-SAME:|-DAG:|-COUNT:|-EMPTY:)"
|
||||
has_arch_prefix
|
||||
"${test_contents}"
|
||||
)
|
||||
if (has_arch_prefix)
|
||||
list(APPEND check_prefixes "${arch_prefix}")
|
||||
endif()
|
||||
|
||||
if (arch MATCHES "^1[0-9][0-9][af]?$")
|
||||
string(
|
||||
REGEX MATCH
|
||||
"; SM1XX(:|-LABEL:|-NOT:|-NEXT:|-SAME:|-DAG:|-COUNT:|-EMPTY:)"
|
||||
has_sm1xx_prefix
|
||||
"${test_contents}"
|
||||
)
|
||||
if (has_sm1xx_prefix)
|
||||
list(APPEND check_prefixes SM1XX)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
string(
|
||||
REGEX MATCHALL
|
||||
"; SM[0-9]+-PLUS(:|-[A-Z]+:)"
|
||||
plus_prefixes
|
||||
"${test_contents}"
|
||||
)
|
||||
foreach (plus_prefix IN LISTS plus_prefixes)
|
||||
string(REGEX REPLACE ".*SM([0-9]+)-PLUS.*" "\\1" plus_arch "${plus_prefix}")
|
||||
if (arch GREATER_EQUAL plus_arch)
|
||||
list(APPEND check_prefixes "SM${plus_arch}-PLUS")
|
||||
endif()
|
||||
endforeach()
|
||||
list(REMOVE_DUPLICATES check_prefixes)
|
||||
list(JOIN check_prefixes "," check_prefixes)
|
||||
|
||||
set(${out_var} "${check_prefixes}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
function(simd_codegen_has_sass_arch_specific_checks out_var test_contents)
|
||||
string(
|
||||
REGEX MATCH
|
||||
"; SM([0-9]+[a-f]?|1XX)(:|-LABEL:|-NOT:|-NEXT:|-SAME:|-DAG:|-COUNT:|-EMPTY:)"
|
||||
has_arch_specific_checks
|
||||
"${test_contents}"
|
||||
)
|
||||
string(
|
||||
REGEX MATCH
|
||||
"; SM[0-9]+-PLUS(:|-[A-Z]+:)"
|
||||
has_plus_checks
|
||||
"${test_contents}"
|
||||
)
|
||||
if (has_arch_specific_checks OR has_plus_checks)
|
||||
set(${out_var} TRUE PARENT_SCOPE)
|
||||
else()
|
||||
set(${out_var} FALSE PARENT_SCOPE)
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
# configure the library target for the test
|
||||
function(simd_codegen_add_library_target out_var kind arch test_path)
|
||||
cmake_path(GET test_path FILENAME test_file)
|
||||
cmake_path(REMOVE_EXTENSION test_file LAST_ONLY OUTPUT_VARIABLE test_name)
|
||||
|
||||
set(target_name "simd_codegen_${kind}_sm${arch}_${test_name}")
|
||||
|
||||
add_library(${target_name} STATIC "${test_path}")
|
||||
|
||||
simd_codegen_set_cuda_arch(${target_name} "${arch}")
|
||||
target_compile_options(${target_name} PRIVATE "-Wno-comment")
|
||||
|
||||
target_include_directories(
|
||||
${target_name}
|
||||
PRIVATE "${libcudacxx_SOURCE_DIR}/include"
|
||||
)
|
||||
set(${out_var} "${target_name}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
# generate the check target for the test (FILECHECK)
|
||||
function(
|
||||
simd_codegen_add_check_target
|
||||
aggregate_target
|
||||
target_name
|
||||
test_path
|
||||
check_prefixes
|
||||
)
|
||||
string(REGEX REPLACE "[^A-Za-z0-9_]" "_" check_suffix "${check_prefixes}")
|
||||
set(check_target_name "${target_name}_${check_suffix}_check")
|
||||
|
||||
add_custom_target(
|
||||
${check_target_name}
|
||||
DEPENDS ${target_name}
|
||||
# gersemi: off
|
||||
COMMAND
|
||||
${CMAKE_COMMAND} -E env "FILECHECK=${filecheck}"
|
||||
"${dump_and_check_script}"
|
||||
$<TARGET_FILE:${target_name}>
|
||||
"${test_path}"
|
||||
"${check_prefixes}"
|
||||
${ARGN}
|
||||
# gersemi: on
|
||||
)
|
||||
add_dependencies(${aggregate_target} ${check_target_name})
|
||||
endfunction()
|
||||
|
||||
# generate the command to run the PTX tests
|
||||
function(simd_codegen_add_ptx_tests arch prefix_list test_list)
|
||||
foreach (test_path IN LISTS ${test_list})
|
||||
simd_codegen_add_library_target(target_name ptx "${arch}" "${test_path}")
|
||||
|
||||
# Clang stopped emitting PTX in clang20. Add flags to re-enable it.
|
||||
if (
|
||||
CMAKE_CUDA_COMPILER_ID STREQUAL Clang
|
||||
AND CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 20
|
||||
)
|
||||
target_compile_options(
|
||||
${target_name}
|
||||
PRIVATE "--cuda-include-ptx=sm_${arch}"
|
||||
)
|
||||
endif()
|
||||
|
||||
foreach (prefix IN LISTS ${prefix_list})
|
||||
simd_codegen_add_check_target(libcudacxx.test.simd.ptx ${target_name} "${test_path}" "${prefix}")
|
||||
endforeach()
|
||||
endforeach()
|
||||
endfunction()
|
||||
|
||||
# generate the command to run the SASS tests
|
||||
function(simd_codegen_add_sass_test test_path)
|
||||
file(READ "${test_path}" test_contents)
|
||||
|
||||
set(test_archs)
|
||||
foreach (arch IN LISTS simd_codegen_sass_cuda_archs)
|
||||
simd_codegen_get_sass_check_prefixes(check_prefixes "${test_contents}" "${arch}")
|
||||
if (NOT "${check_prefixes}" STREQUAL "SMXX")
|
||||
list(APPEND test_archs "${arch}")
|
||||
endif()
|
||||
endforeach()
|
||||
if (NOT test_archs)
|
||||
simd_codegen_has_sass_arch_specific_checks(has_arch_specific_checks "${test_contents}")
|
||||
if (has_arch_specific_checks)
|
||||
message(
|
||||
STATUS
|
||||
"-- Skipping ${test_path}: requires an unsupported CUDA architecture"
|
||||
)
|
||||
return()
|
||||
else()
|
||||
set(test_archs ${simd_codegen_sass_cuda_archs})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
foreach (arch IN LISTS test_archs)
|
||||
simd_codegen_add_library_target(target_name sass "${arch}" "${test_path}")
|
||||
simd_codegen_get_sass_check_prefixes(check_prefixes "${test_contents}" "${arch}")
|
||||
|
||||
string(FIND "${test_contents}" "__device__" has_device_function)
|
||||
if (NOT has_device_function EQUAL -1)
|
||||
target_compile_options(${target_name} PRIVATE "-dc")
|
||||
endif()
|
||||
|
||||
simd_codegen_add_check_target(libcudacxx.test.simd.sass ${target_name} "${test_path}" "${check_prefixes}" --dump-sass)
|
||||
endforeach()
|
||||
endfunction()
|
||||
|
||||
set(simd_codegen_sass_tests)
|
||||
file(
|
||||
GLOB simd_codegen_sass_tests
|
||||
"floating_point/*.cu"
|
||||
"integer/*.cu"
|
||||
"min_max/*.cu"
|
||||
)
|
||||
|
||||
add_subdirectory(load_store)
|
||||
|
||||
foreach (test_path IN LISTS simd_codegen_sass_tests)
|
||||
simd_codegen_add_sass_test("${test_path}")
|
||||
endforeach()
|
||||
@@ -0,0 +1,31 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_f32_x4 = simd::basic_vec<float, simd::fixed_size<4>>;
|
||||
|
||||
__device__ Vec_f32_x4 test_operator_decrement_f32_x4(Vec_f32_x4 vec)
|
||||
{
|
||||
--vec;
|
||||
return vec;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_decrement_f32_x4.*}}
|
||||
; SM100: {{.*FADD2.*}}
|
||||
; SM100: {{.*FADD2.*}}
|
||||
; SM103: {{.*FADD2.*}}
|
||||
; SM103: {{.*FADD2.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,38 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
#if _CCCL_HAS_NVBF16()
|
||||
|
||||
# include <cuda_bf16.h>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_bf16_x4 = simd::basic_vec<__nv_bfloat16, simd::fixed_size<4>>;
|
||||
|
||||
__device__ Vec_bf16_x4 test_fma_bf16_x4(Vec_bf16_x4 lhs, Vec_bf16_x4 rhs, Vec_bf16_x4 add)
|
||||
{
|
||||
return lhs * rhs + add;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_fma_bf16_x4.*}}
|
||||
; SM80: {{.*HFMA2.*BF16.*}}
|
||||
; SM80: {{.*HFMA2.*BF16.*}}
|
||||
; SM90: {{.*HFMA2.*BF16.*}}
|
||||
; SM90: {{.*HFMA2.*BF16.*}}
|
||||
; SM1XX: {{.*HFMA2.*BF16.*}}
|
||||
; SM1XX: {{.*HFMA2.*BF16.*}}
|
||||
|
||||
*/
|
||||
|
||||
#endif // _CCCL_HAS_NVBF16()
|
||||
@@ -0,0 +1,38 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
#if _CCCL_HAS_NVFP16()
|
||||
|
||||
# include <cuda_fp16.h>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_f16_x4 = simd::basic_vec<__half, simd::fixed_size<4>>;
|
||||
|
||||
__device__ Vec_f16_x4 test_fma_f16_x4(Vec_f16_x4 lhs, Vec_f16_x4 rhs, Vec_f16_x4 add)
|
||||
{
|
||||
return lhs * rhs + add;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_fma_f16_x4.*}}
|
||||
; SM80: {{.*HFMA2.*}}
|
||||
; SM80: {{.*HFMA2.*}}
|
||||
; SM90: {{.*HFMA2.*}}
|
||||
; SM90: {{.*HFMA2.*}}
|
||||
; SM1XX: {{.*HFMA2.*}}
|
||||
; SM1XX: {{.*HFMA2.*}}
|
||||
|
||||
*/
|
||||
|
||||
#endif // _CCCL_HAS_NVFP16()
|
||||
@@ -0,0 +1,30 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_f32_x4 = simd::basic_vec<float, simd::fixed_size<4>>;
|
||||
|
||||
__device__ Vec_f32_x4 test_fma_f32_x4(Vec_f32_x4 lhs, Vec_f32_x4 rhs, Vec_f32_x4 add)
|
||||
{
|
||||
return simd::fma(lhs, rhs, add);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_fma_f32_x4.*}}
|
||||
; SM100: {{.*FFMA2.*}}
|
||||
; SM100: {{.*FFMA2.*}}
|
||||
; SM103: {{.*FFMA2.*}}
|
||||
; SM103: {{.*FFMA2.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,31 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_f32_x4 = simd::basic_vec<float, simd::fixed_size<4>>;
|
||||
|
||||
__device__ Vec_f32_x4 test_operator_increment_f32_x4(Vec_f32_x4 vec)
|
||||
{
|
||||
++vec;
|
||||
return vec;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_increment_f32_x4.*}}
|
||||
; SM100: {{.*FADD2.*}}
|
||||
; SM100: {{.*FADD2.*}}
|
||||
; SM103: {{.*FADD2.*}}
|
||||
; SM103: {{.*FADD2.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,41 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
#if _CCCL_HAS_NVBF16()
|
||||
|
||||
# include <cuda_bf16.h>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_bf16_x4 = simd::basic_vec<__nv_bfloat16, simd::fixed_size<4>>;
|
||||
using Mask_bf16_x4 = Vec_bf16_x4::mask_type;
|
||||
|
||||
__device__ Mask_bf16_x4 test_less_bf16_x4(Vec_bf16_x4 lhs, Vec_bf16_x4 rhs)
|
||||
{
|
||||
return lhs < rhs;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_less_bf16_x4.*}}
|
||||
; SM80: {{.*FSETP\.LT.*}}
|
||||
; SM80: {{.*FSETP\.LT.*}}
|
||||
; SM80: {{.*FSETP\.LT.*}}
|
||||
; SM80: {{.*FSETP\.LT.*}}
|
||||
; SM90: {{.*HSETP2.*BF16.*}}
|
||||
; SM90: {{.*HSETP2.*BF16.*}}
|
||||
; SM1XX: {{.*HSETP2.*BF16.*}}
|
||||
; SM1XX: {{.*HSETP2.*BF16.*}}
|
||||
|
||||
*/
|
||||
|
||||
#endif // _CCCL_HAS_NVBF16()
|
||||
@@ -0,0 +1,39 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
#if _CCCL_HAS_NVFP16()
|
||||
|
||||
# include <cuda_fp16.h>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_f16_x4 = simd::basic_vec<__half, simd::fixed_size<4>>;
|
||||
using Mask_f16_x4 = Vec_f16_x4::mask_type;
|
||||
|
||||
__device__ Mask_f16_x4 test_less_f16_x4(Vec_f16_x4 lhs, Vec_f16_x4 rhs)
|
||||
{
|
||||
return lhs < rhs;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_less_f16_x4.*}}
|
||||
; SM80: {{.*HSETP2.*}}
|
||||
; SM80: {{.*HSETP2.*}}
|
||||
; SM90: {{.*HSETP2.*}}
|
||||
; SM90: {{.*HSETP2.*}}
|
||||
; SM1XX: {{.*HSETP2.*}}
|
||||
; SM1XX: {{.*HSETP2.*}}
|
||||
|
||||
*/
|
||||
|
||||
#endif // _CCCL_HAS_NVFP16()
|
||||
@@ -0,0 +1,30 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_f32_x4 = simd::basic_vec<float, simd::fixed_size<4>>;
|
||||
|
||||
__device__ Vec_f32_x4 test_operator_minus_f32_x4(Vec_f32_x4 lhs, Vec_f32_x4 rhs)
|
||||
{
|
||||
return lhs - rhs;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_minus_f32_x4.*}}
|
||||
; SM100: {{.*FADD2.*}}
|
||||
; SM100: {{.*FADD2.*}}
|
||||
; SM103: {{.*FADD2.*}}
|
||||
; SM103: {{.*FADD2.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,38 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
#if _CCCL_HAS_NVBF16()
|
||||
|
||||
# include <cuda_bf16.h>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_bf16_x4 = simd::basic_vec<__nv_bfloat16, simd::fixed_size<4>>;
|
||||
|
||||
__device__ Vec_bf16_x4 test_operator_multiplies_bf16_x4(Vec_bf16_x4 lhs, Vec_bf16_x4 rhs)
|
||||
{
|
||||
return lhs * rhs;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_multiplies_bf16_x4.*}}
|
||||
; SM80: {{.*HFMA2.*BF16.*}}
|
||||
; SM80: {{.*HFMA2.*BF16.*}}
|
||||
; SM90: {{.*HFMA2.*BF16.*}}
|
||||
; SM90: {{.*HMUL2.*BF16.*}}
|
||||
; SM1XX: {{.*HFMA2.*BF16.*}}
|
||||
; SM1XX: {{.*HMUL2.*BF16.*}}
|
||||
|
||||
*/
|
||||
|
||||
#endif // _CCCL_HAS_NVBF16()
|
||||
@@ -0,0 +1,38 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
#if _CCCL_HAS_NVFP16()
|
||||
|
||||
# include <cuda_fp16.h>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_f16_x4 = simd::basic_vec<__half, simd::fixed_size<4>>;
|
||||
|
||||
__device__ Vec_f16_x4 test_operator_multiplies_f16_x4(Vec_f16_x4 lhs, Vec_f16_x4 rhs)
|
||||
{
|
||||
return lhs * rhs;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_multiplies_f16_x4.*}}
|
||||
; SM80: {{.*(HMUL2|HFMA2).*}}
|
||||
; SM80: {{.*(HMUL2|HFMA2).*}}
|
||||
; SM90: {{.*(HMUL2|HFMA2).*}}
|
||||
; SM90: {{.*(HMUL2|HFMA2).*}}
|
||||
; SM1XX: {{.*(HMUL2|HFMA2).*}}
|
||||
; SM1XX: {{.*(HMUL2|HFMA2).*}}
|
||||
|
||||
*/
|
||||
|
||||
#endif // _CCCL_HAS_NVFP16()
|
||||
@@ -0,0 +1,38 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
#if _CCCL_HAS_NVBF16()
|
||||
|
||||
# include <cuda_bf16.h>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_bf16_x4 = simd::basic_vec<__nv_bfloat16, simd::fixed_size<4>>;
|
||||
|
||||
__device__ Vec_bf16_x4 test_operator_plus_bf16_x4(Vec_bf16_x4 lhs, Vec_bf16_x4 rhs)
|
||||
{
|
||||
return lhs + rhs;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_plus_bf16_x4.*}}
|
||||
; SM80: {{.*HFMA2.*BF16.*}}
|
||||
; SM80: {{.*HFMA2.*BF16.*}}
|
||||
; SM90: {{.*HFMA2.*BF16.*}}
|
||||
; SM90: {{.*HADD2.*BF16.*}}
|
||||
; SM1XX: {{.*HFMA2.*BF16.*}}
|
||||
; SM1XX: {{.*HADD2.*BF16.*}}
|
||||
|
||||
*/
|
||||
|
||||
#endif // _CCCL_HAS_NVBF16()
|
||||
@@ -0,0 +1,34 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
#if _CCCL_HAS_NVFP16()
|
||||
|
||||
# include <cuda_fp16.h>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_f16_x4 = simd::basic_vec<__half, simd::fixed_size<4>>;
|
||||
|
||||
__device__ Vec_f16_x4 test_operator_plus_f16_x4(Vec_f16_x4 lhs, Vec_f16_x4 rhs)
|
||||
{
|
||||
return lhs + rhs;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_plus_f16_x4.*}}
|
||||
; SMXX: {{.*(HADD2|HFMA2).*}}
|
||||
; SMXX: {{.*(HADD2|HFMA2).*}}
|
||||
|
||||
*/
|
||||
|
||||
#endif // _CCCL_HAS_NVFP16()
|
||||
@@ -0,0 +1,30 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_f32_x4 = simd::basic_vec<float, simd::fixed_size<4>>;
|
||||
|
||||
__device__ Vec_f32_x4 test_operator_plus_f32_x4(Vec_f32_x4 lhs, Vec_f32_x4 rhs)
|
||||
{
|
||||
return lhs + rhs;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_plus_f32_x4.*}}
|
||||
; SM100: {{.*FADD2.*}}
|
||||
; SM100: {{.*FADD2.*}}
|
||||
; SM103: {{.*FADD2.*}}
|
||||
; SM103: {{.*FADD2.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,30 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_f32_x4 = simd::basic_vec<float, simd::fixed_size<4>>;
|
||||
|
||||
__device__ Vec_f32_x4 test_operator_unary_minus_f32_x4(Vec_f32_x4 in)
|
||||
{
|
||||
return -in;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_unary_minus_f32_x4.*}}
|
||||
; SM100: {{.*FADD2.*}}
|
||||
; SM100: {{.*FADD2.*}}
|
||||
; SM103: {{.*FADD2.*}}
|
||||
; SM103: {{.*FADD2.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,66 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_u16_x2 = simd::basic_vec<cuda::std::uint16_t, simd::fixed_size<2>>;
|
||||
|
||||
__device__ Vec_u16_x2 test_operator_plus_u16_x2(Vec_u16_x2 lhs, Vec_u16_x2 rhs)
|
||||
{
|
||||
return lhs + rhs;
|
||||
}
|
||||
|
||||
__device__ Vec_u16_x2 test_operator_minus_u16_x2(Vec_u16_x2 lhs, Vec_u16_x2 rhs)
|
||||
{
|
||||
return lhs - rhs;
|
||||
}
|
||||
|
||||
__device__ Vec_u16_x2 test_operator_post_decrement_u16_x2(Vec_u16_x2 in)
|
||||
{
|
||||
(void) in--;
|
||||
return in;
|
||||
}
|
||||
|
||||
__device__ Vec_u16_x2 test_operator_post_increment_u16_x2(Vec_u16_x2 in)
|
||||
{
|
||||
(void) in++;
|
||||
return in;
|
||||
}
|
||||
|
||||
__device__ Vec_u16_x2 test_operator_unary_minus_u16_x2(Vec_u16_x2 in)
|
||||
{
|
||||
return -in;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_unary_minus_u16_x2.*}}
|
||||
; SM90: {{.*VIADD.*}}
|
||||
; SM1XX: {{.*VIADD.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_post_increment_u16_x2.*}}
|
||||
; SM90: {{.*VIADD.*}}
|
||||
; SM1XX: {{.*VIADD.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_post_decrement_u16_x2.*}}
|
||||
; SM90: {{.*VIADD.*}}
|
||||
; SM1XX: {{.*VIADD.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_minus_u16_x2.*}}
|
||||
; SM90: {{.*VIADD.*}}
|
||||
; SM1XX: {{.*VIADD.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_plus_u16_x2.*}}
|
||||
; SM90: {{.*VIADD.*}}
|
||||
; SM1XX: {{.*VIADD.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,61 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_u8_x4 = simd::basic_vec<cuda::std::uint8_t, simd::fixed_size<4>>;
|
||||
|
||||
__device__ Vec_u8_x4 test_operator_plus_u8_x4(Vec_u8_x4 lhs, Vec_u8_x4 rhs)
|
||||
{
|
||||
return lhs + rhs;
|
||||
}
|
||||
|
||||
__device__ Vec_u8_x4 test_operator_minus_u8_x4(Vec_u8_x4 lhs, Vec_u8_x4 rhs)
|
||||
{
|
||||
return lhs - rhs;
|
||||
}
|
||||
|
||||
__device__ Vec_u8_x4 test_operator_post_decrement_u8_x4(Vec_u8_x4 in)
|
||||
{
|
||||
(void) in--;
|
||||
return in;
|
||||
}
|
||||
|
||||
__device__ Vec_u8_x4 test_operator_post_increment_u8_x4(Vec_u8_x4 in)
|
||||
{
|
||||
(void) in++;
|
||||
return in;
|
||||
}
|
||||
|
||||
__device__ Vec_u8_x4 test_operator_unary_minus_u8_x4(Vec_u8_x4 in)
|
||||
{
|
||||
return -in;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_unary_minus_u8_x4.*}}
|
||||
; SM120f: {{.*VIADD.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_post_increment_u8_x4.*}}
|
||||
; SM120f: {{.*VIADD.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_post_decrement_u8_x4.*}}
|
||||
; SM120f: {{.*VIADD.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_minus_u8_x4.*}}
|
||||
; SM120f: {{.*VIADD.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_operator_plus_u8_x4.*}}
|
||||
; SM120f: {{.*VIADD.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,95 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using cuda::std::uint16_t;
|
||||
using cuda::std::uint8_t;
|
||||
|
||||
using Vec_u16_x2 = simd::basic_vec<uint16_t, simd::fixed_size<2>>;
|
||||
using Vec_u8_x4 = simd::basic_vec<uint8_t, simd::fixed_size<4>>;
|
||||
|
||||
__device__ void test_bitwise_and_u16_x2(Vec_u16_x2& out, Vec_u16_x2& lhs, Vec_u16_x2& rhs)
|
||||
{
|
||||
out = lhs & rhs;
|
||||
}
|
||||
|
||||
__device__ void test_bitwise_or_u16_x2(Vec_u16_x2& out, Vec_u16_x2& lhs, Vec_u16_x2& rhs)
|
||||
{
|
||||
out = lhs | rhs;
|
||||
}
|
||||
|
||||
__device__ void test_bitwise_xor_u16_x2(Vec_u16_x2& out, Vec_u16_x2& lhs, Vec_u16_x2& rhs)
|
||||
{
|
||||
out = lhs ^ rhs;
|
||||
}
|
||||
|
||||
__device__ void test_bitwise_not_u16_x2(Vec_u16_x2& out, Vec_u16_x2& in)
|
||||
{
|
||||
out = ~in;
|
||||
}
|
||||
|
||||
__device__ void test_bitwise_and_u8_x4(Vec_u8_x4& out, Vec_u8_x4& lhs, Vec_u8_x4& rhs)
|
||||
{
|
||||
out = lhs & rhs;
|
||||
}
|
||||
|
||||
__device__ void test_bitwise_or_u8_x4(Vec_u8_x4& out, Vec_u8_x4& lhs, Vec_u8_x4& rhs)
|
||||
{
|
||||
out = lhs | rhs;
|
||||
}
|
||||
|
||||
__device__ void test_bitwise_xor_u8_x4(Vec_u8_x4& out, Vec_u8_x4& lhs, Vec_u8_x4& rhs)
|
||||
{
|
||||
out = lhs ^ rhs;
|
||||
}
|
||||
|
||||
__device__ void test_bitwise_not_u8_x4(Vec_u8_x4& out, Vec_u8_x4& in)
|
||||
{
|
||||
out = ~in;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_bitwise_not_u8_x4.*}}
|
||||
; SMXX-NOT: {{.*(LD\.E\.U(8|16)|PRMT|SHF|I2I|IMAD\.SHL).*}}
|
||||
; SMXX: {{.*LOP3.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_bitwise_xor_u8_x4.*}}
|
||||
; SMXX-NOT: {{.*(LD\.E\.U(8|16)|PRMT|SHF|I2I|IMAD\.SHL).*}}
|
||||
; SMXX: {{.*LOP3.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_bitwise_or_u8_x4.*}}
|
||||
; SMXX-NOT: {{.*(LD\.E\.U(8|16)|PRMT|SHF|I2I|IMAD\.SHL).*}}
|
||||
; SMXX: {{.*LOP3.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_bitwise_and_u8_x4.*}}
|
||||
; SMXX-NOT: {{.*(LD\.E\.U(8|16)|PRMT|SHF|I2I|IMAD\.SHL).*}}
|
||||
; SMXX: {{.*LOP3.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_bitwise_not_u16_x2.*}}
|
||||
; SMXX-NOT: {{.*(LD\.E\.U(8|16)|PRMT|SHF|I2I|IMAD\.SHL).*}}
|
||||
; SMXX: {{.*LOP3.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_bitwise_xor_u16_x2.*}}
|
||||
; SMXX-NOT: {{.*(LD\.E\.U(8|16)|PRMT|SHF|I2I|IMAD\.SHL).*}}
|
||||
; SMXX: {{.*LOP3.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_bitwise_or_u16_x2.*}}
|
||||
; SMXX-NOT: {{.*(LD\.E\.U(8|16)|PRMT|SHF|I2I|IMAD\.SHL).*}}
|
||||
; SMXX: {{.*LOP3.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_bitwise_and_u16_x2.*}}
|
||||
; SMXX-NOT: {{.*(LD\.E\.U(8|16)|PRMT|SHF|I2I|IMAD\.SHL).*}}
|
||||
; SMXX: {{.*LOP3.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,35 @@
|
||||
##===----------------------------------------------------------------------===##
|
||||
##
|
||||
## Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
## under the Apache License v2.0 with LLVM Exceptions.
|
||||
## See https://llvm.org/LICENSE.txt for license information.
|
||||
## SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
## SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
##
|
||||
##===----------------------------------------------------------------------===##
|
||||
|
||||
set(
|
||||
simd_codegen_load_store_tests
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/load_store_i8.cu"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/load_store_i16.cu"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/load_store_i32.cu"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/load_store_f16.cu"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/load_store_bf16.cu"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/load_store_f32.cu"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/load_store_i64.cu"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/load_store_f64.cu"
|
||||
)
|
||||
|
||||
set(simd_codegen_default_prefixes SMXXX)
|
||||
simd_codegen_add_ptx_tests(80 simd_codegen_default_prefixes simd_codegen_load_store_tests)
|
||||
simd_codegen_add_ptx_tests(90 simd_codegen_default_prefixes simd_codegen_load_store_tests)
|
||||
|
||||
# SM100/SM120 add 32-byte vectorized checks.
|
||||
if (
|
||||
CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.9
|
||||
AND NOT CMAKE_CUDA_COMPILER_ID STREQUAL Clang
|
||||
)
|
||||
set(simd_codegen_sm100_prefixes SMXXX SM100-PLUS)
|
||||
simd_codegen_add_ptx_tests(100 simd_codegen_sm100_prefixes simd_codegen_load_store_tests)
|
||||
simd_codegen_add_ptx_tests(120 simd_codegen_sm100_prefixes simd_codegen_load_store_tests)
|
||||
endif()
|
||||
@@ -0,0 +1,54 @@
|
||||
#include <cuda/std/__simd/load.h>
|
||||
#include <cuda/std/__simd/store.h>
|
||||
|
||||
#if _CCCL_HAS_NVBF16()
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
// --- 8-byte tier: __nv_bfloat16, N=4 ---
|
||||
|
||||
using Vec_bf16_4 = simd::basic_vec<__nv_bfloat16, simd::fixed_size<4>>;
|
||||
|
||||
__global__ void test_load_bf16_4(const __nv_bfloat16* in, __nv_bfloat16* out)
|
||||
{
|
||||
Vec_bf16_4 v = simd::unchecked_load<Vec_bf16_4>(in, Vec_bf16_4::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_bf16_4::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
// --- 16-byte tier: __nv_bfloat16, N=8 ---
|
||||
|
||||
using Vec_bf16_8 = simd::basic_vec<__nv_bfloat16, simd::fixed_size<8>>;
|
||||
|
||||
__global__ void test_load_bf16_8(const __nv_bfloat16* in, __nv_bfloat16* out)
|
||||
{
|
||||
Vec_bf16_8 v = simd::unchecked_load<Vec_bf16_8>(in, Vec_bf16_8::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_bf16_8::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
// --- 32-byte tier: __nv_bfloat16, N=16 ---
|
||||
|
||||
using Vec_bf16_16 = simd::basic_vec<__nv_bfloat16, simd::fixed_size<16>>;
|
||||
|
||||
__global__ void test_load_bf16_16(const __nv_bfloat16* in, __nv_bfloat16* out)
|
||||
{
|
||||
Vec_bf16_16 v = simd::unchecked_load<Vec_bf16_16>(in, Vec_bf16_16::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_bf16_16::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXXX-LABEL: .visible .entry {{.*}}test_load_bf16_4{{.*}}(
|
||||
; SMXXX: {{.*}}ld.global.{{([bus]64|v4\.[bus]16|v2\.[bus]32)}}{{.*}}
|
||||
; SMXXX: {{.*}}st.global.{{([bus]64|v4\.[bus]16|v2\.[bus]32)}}{{.*}}
|
||||
|
||||
; SMXXX-LABEL: .visible .entry {{.*}}test_load_bf16_8{{.*}}(
|
||||
; SMXXX: {{.*}}ld.global.{{(v4\.[bus]32|v2\.[bus]64)}}{{.*}}
|
||||
; SMXXX: {{.*}}st.global.{{(v4\.[bus]32|v2\.[bus]64)}}{{.*}}
|
||||
|
||||
; SM100-PLUS-LABEL: .visible .entry {{.*}}test_load_bf16_16{{.*}}(
|
||||
; SM100-PLUS: {{.*}}ld.global.{{(v4\.[bus]64|v8\.[bus]32)}}{{.*}}
|
||||
; SM100-PLUS: {{.*}}st.global.{{(v4\.[bus]64|v8\.[bus]32)}}{{.*}}
|
||||
|
||||
*/
|
||||
|
||||
#endif // _CCCL_HAS_NVBF16()
|
||||
@@ -0,0 +1,54 @@
|
||||
#include <cuda/std/__simd/load.h>
|
||||
#include <cuda/std/__simd/store.h>
|
||||
|
||||
#if _CCCL_HAS_NVFP16()
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
// --- 8-byte tier: __half, N=4 ---
|
||||
|
||||
using Vec_f16_4 = simd::basic_vec<__half, simd::fixed_size<4>>;
|
||||
|
||||
__global__ void test_load_f16_4(const __half* in, __half* out)
|
||||
{
|
||||
Vec_f16_4 v = simd::unchecked_load<Vec_f16_4>(in, Vec_f16_4::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_f16_4::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
// --- 16-byte tier: __half, N=8 ---
|
||||
|
||||
using Vec_f16_8 = simd::basic_vec<__half, simd::fixed_size<8>>;
|
||||
|
||||
__global__ void test_load_f16_8(const __half* in, __half* out)
|
||||
{
|
||||
Vec_f16_8 v = simd::unchecked_load<Vec_f16_8>(in, Vec_f16_8::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_f16_8::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
// --- 32-byte tier: __half, N=16 ---
|
||||
|
||||
using Vec_f16_16 = simd::basic_vec<__half, simd::fixed_size<16>>;
|
||||
|
||||
__global__ void test_load_f16_16(const __half* in, __half* out)
|
||||
{
|
||||
Vec_f16_16 v = simd::unchecked_load<Vec_f16_16>(in, Vec_f16_16::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_f16_16::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXXX-LABEL: .visible .entry {{.*}}test_load_f16_4{{.*}}(
|
||||
; SMXXX: {{.*}}ld.global.{{([bus]64|v4\.[bfus]16|v2\.[bus]32)}}{{.*}}
|
||||
; SMXXX: {{.*}}st.global.{{([bus]64|v4\.[bfus]16|v2\.[bus]32)}}{{.*}}
|
||||
|
||||
; SMXXX-LABEL: .visible .entry {{.*}}test_load_f16_8{{.*}}(
|
||||
; SMXXX: {{.*}}ld.global.{{(v4\.[bus]32|v2\.[bus]64)}}{{.*}}
|
||||
; SMXXX: {{.*}}st.global.{{(v4\.[bus]32|v2\.[bus]64)}}{{.*}}
|
||||
|
||||
; SM100-PLUS-LABEL: .visible .entry {{.*}}test_load_f16_16{{.*}}(
|
||||
; SM100-PLUS: {{.*}}ld.global.{{(v4\.[bus]64|v8\.[bus]32)}}{{.*}}
|
||||
; SM100-PLUS: {{.*}}st.global.{{(v4\.[bus]64|v8\.[bus]32)}}{{.*}}
|
||||
|
||||
*/
|
||||
|
||||
#endif // _CCCL_HAS_NVFP16()
|
||||
@@ -0,0 +1,50 @@
|
||||
#include <cuda/std/__simd/load.h>
|
||||
#include <cuda/std/__simd/store.h>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
// --- 8-byte tier: float, N=2 ---
|
||||
|
||||
using Vec_f32_2 = simd::basic_vec<float, simd::fixed_size<2>>;
|
||||
|
||||
__global__ void test_load_f32_2(const float* in, float* out)
|
||||
{
|
||||
Vec_f32_2 v = simd::unchecked_load<Vec_f32_2>(in, Vec_f32_2::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_f32_2::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
// --- 16-byte tier: float, N=4 ---
|
||||
|
||||
using Vec_f32_4 = simd::basic_vec<float, simd::fixed_size<4>>;
|
||||
|
||||
__global__ void test_load_f32_4(const float* in, float* out)
|
||||
{
|
||||
Vec_f32_4 v = simd::unchecked_load<Vec_f32_4>(in, Vec_f32_4::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_f32_4::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
// --- 32-byte tier: float, N=8 ---
|
||||
|
||||
using Vec_f32_8 = simd::basic_vec<float, simd::fixed_size<8>>;
|
||||
|
||||
__global__ void test_load_f32_8(const float* in, float* out)
|
||||
{
|
||||
Vec_f32_8 v = simd::unchecked_load<Vec_f32_8>(in, Vec_f32_8::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_f32_8::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXXX-LABEL: .visible .entry {{.*}}test_load_f32_2{{.*}}(
|
||||
; SMXXX: {{.*}}ld.global.{{([bus]64|v4\.[bus]16|v2\.[bfus]32)}}{{.*}}
|
||||
; SMXXX: {{.*}}st.global.{{([bus]64|v4\.[bus]16|v2\.[bfus]32)}}{{.*}}
|
||||
|
||||
; SMXXX-LABEL: .visible .entry {{.*}}test_load_f32_4{{.*}}(
|
||||
; SMXXX: {{.*}}ld.global.{{(v4\.[bfus]32|v2\.[bus]64)}}{{.*}}
|
||||
; SMXXX: {{.*}}st.global.{{(v4\.[bfus]32|v2\.[bus]64)}}{{.*}}
|
||||
|
||||
; SM100-PLUS-LABEL: .visible .entry {{.*}}test_load_f32_8{{.*}}(
|
||||
; SM100-PLUS: {{.*}}ld.global.{{(v4\.[bus]64|v8\.[bfus]32)}}{{.*}}
|
||||
; SM100-PLUS: {{.*}}st.global.{{(v4\.[bus]64|v8\.[bfus]32)}}{{.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,37 @@
|
||||
#include <cuda/std/__simd/load.h>
|
||||
#include <cuda/std/__simd/store.h>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
// --- 16-byte tier: double, N=2 ---
|
||||
// 8-byte tier skipped: N=1
|
||||
|
||||
using Vec_f64_2 = simd::basic_vec<double, simd::fixed_size<2>>;
|
||||
|
||||
__global__ void test_load_f64_2(const double* in, double* out)
|
||||
{
|
||||
Vec_f64_2 v = simd::unchecked_load<Vec_f64_2>(in, Vec_f64_2::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_f64_2::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
// --- 32-byte tier: double, N=4 ---
|
||||
|
||||
using Vec_f64_4 = simd::basic_vec<double, simd::fixed_size<4>>;
|
||||
|
||||
__global__ void test_load_f64_4(const double* in, double* out)
|
||||
{
|
||||
Vec_f64_4 v = simd::unchecked_load<Vec_f64_4>(in, Vec_f64_4::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_f64_4::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXXX-LABEL: .visible .entry {{.*}}test_load_f64_2{{.*}}(
|
||||
; SMXXX: {{.*}}ld.global.{{(v4\.[bus]32|v2\.[bfus]64)}}{{.*}}
|
||||
; SMXXX: {{.*}}st.global.{{(v4\.[bus]32|v2\.[bfus]64)}}{{.*}}
|
||||
|
||||
; SM100-PLUS-LABEL: .visible .entry {{.*}}test_load_f64_4{{.*}}(
|
||||
; SM100-PLUS: {{.*}}ld.global.{{(v4\.[bfus]64|v8\.[bus]32)}}{{.*}}
|
||||
; SM100-PLUS: {{.*}}st.global.{{(v4\.[bfus]64|v8\.[bus]32)}}{{.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,51 @@
|
||||
#include <cuda/std/__simd/load.h>
|
||||
#include <cuda/std/__simd/store.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
// --- 8-byte tier: int16_t, N=4 ---
|
||||
|
||||
using Vec_i16_4 = simd::basic_vec<cuda::std::int16_t, simd::fixed_size<4>>;
|
||||
|
||||
__global__ void test_load_i16_4(const cuda::std::int16_t* in, cuda::std::int16_t* out)
|
||||
{
|
||||
Vec_i16_4 v = simd::unchecked_load<Vec_i16_4>(in, Vec_i16_4::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_i16_4::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
// --- 16-byte tier: int16_t, N=8 ---
|
||||
|
||||
using Vec_i16_8 = simd::basic_vec<cuda::std::int16_t, simd::fixed_size<8>>;
|
||||
|
||||
__global__ void test_load_i16_8(const cuda::std::int16_t* in, cuda::std::int16_t* out)
|
||||
{
|
||||
Vec_i16_8 v = simd::unchecked_load<Vec_i16_8>(in, Vec_i16_8::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_i16_8::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
// --- 32-byte tier: int16_t, N=16 ---
|
||||
|
||||
using Vec_i16_16 = simd::basic_vec<cuda::std::int16_t, simd::fixed_size<16>>;
|
||||
|
||||
__global__ void test_load_i16_16(const cuda::std::int16_t* in, cuda::std::int16_t* out)
|
||||
{
|
||||
Vec_i16_16 v = simd::unchecked_load<Vec_i16_16>(in, Vec_i16_16::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_i16_16::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXXX-LABEL: .visible .entry {{.*}}test_load_i16_4{{.*}}(
|
||||
; SMXXX: {{.*}}ld.global.{{([bus]64|v4\.[bus]16|v2\.[bus]32)}}{{.*}}
|
||||
; SMXXX: {{.*}}st.global.{{([bus]64|v4\.[bus]16|v2\.[bus]32)}}{{.*}}
|
||||
|
||||
; SMXXX-LABEL: .visible .entry {{.*}}test_load_i16_8{{.*}}(
|
||||
; SMXXX: {{.*}}ld.global.{{(v4\.[bus]32|v2\.[bus]64)}}{{.*}}
|
||||
; SMXXX: {{.*}}st.global.{{(v4\.[bus]32|v2\.[bus]64)}}{{.*}}
|
||||
|
||||
; SM100-PLUS-LABEL: .visible .entry {{.*}}test_load_i16_16{{.*}}(
|
||||
; SM100-PLUS: {{.*}}ld.global.{{(v4\.[bus]64|v8\.[bus]32)}}{{.*}}
|
||||
; SM100-PLUS: {{.*}}st.global.{{(v4\.[bus]64|v8\.[bus]32)}}{{.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,51 @@
|
||||
#include <cuda/std/__simd/load.h>
|
||||
#include <cuda/std/__simd/store.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
// --- 8-byte tier: int32_t, N=2 ---
|
||||
|
||||
using Vec_i32_2 = simd::basic_vec<cuda::std::int32_t, simd::fixed_size<2>>;
|
||||
|
||||
__global__ void test_load_i32_2(const cuda::std::int32_t* in, cuda::std::int32_t* out)
|
||||
{
|
||||
Vec_i32_2 v = simd::unchecked_load<Vec_i32_2>(in, Vec_i32_2::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_i32_2::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
// --- 16-byte tier: int32_t, N=4 ---
|
||||
|
||||
using Vec_i32_4 = simd::basic_vec<cuda::std::int32_t, simd::fixed_size<4>>;
|
||||
|
||||
__global__ void test_load_i32_4(const cuda::std::int32_t* in, cuda::std::int32_t* out)
|
||||
{
|
||||
Vec_i32_4 v = simd::unchecked_load<Vec_i32_4>(in, Vec_i32_4::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_i32_4::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
// --- 32-byte tier: int32_t, N=8 ---
|
||||
|
||||
using Vec_i32_8 = simd::basic_vec<cuda::std::int32_t, simd::fixed_size<8>>;
|
||||
|
||||
__global__ void test_load_i32_8(const cuda::std::int32_t* in, cuda::std::int32_t* out)
|
||||
{
|
||||
Vec_i32_8 v = simd::unchecked_load<Vec_i32_8>(in, Vec_i32_8::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_i32_8::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXXX-LABEL: .visible .entry {{.*}}test_load_i32_2{{.*}}(
|
||||
; SMXXX: {{.*}}ld.global.{{([bus]64|v4\.[bus]16|v2\.[bus]32)}}{{.*}}
|
||||
; SMXXX: {{.*}}st.global.{{([bus]64|v4\.[bus]16|v2\.[bus]32)}}{{.*}}
|
||||
|
||||
; SMXXX-LABEL: .visible .entry {{.*}}test_load_i32_4{{.*}}(
|
||||
; SMXXX: {{.*}}ld.global.{{(v4\.[bus]32|v2\.[bus]64)}}{{.*}}
|
||||
; SMXXX: {{.*}}st.global.{{(v4\.[bus]32|v2\.[bus]64)}}{{.*}}
|
||||
|
||||
; SM100-PLUS-LABEL: .visible .entry {{.*}}test_load_i32_8{{.*}}(
|
||||
; SM100-PLUS: {{.*}}ld.global.{{(v4\.[bus]64|v8\.[bus]32)}}{{.*}}
|
||||
; SM100-PLUS: {{.*}}st.global.{{(v4\.[bus]64|v8\.[bus]32)}}{{.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,38 @@
|
||||
#include <cuda/std/__simd/load.h>
|
||||
#include <cuda/std/__simd/store.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
// --- 16-byte tier: int64_t, N=2 ---
|
||||
// 8-byte tier skipped: N=1
|
||||
|
||||
using Vec_i64_2 = simd::basic_vec<cuda::std::int64_t, simd::fixed_size<2>>;
|
||||
|
||||
__global__ void test_load_i64_2(const cuda::std::int64_t* in, cuda::std::int64_t* out)
|
||||
{
|
||||
Vec_i64_2 v = simd::unchecked_load<Vec_i64_2>(in, Vec_i64_2::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_i64_2::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
// --- 32-byte tier: int64_t, N=4 ---
|
||||
|
||||
using Vec_i64_4 = simd::basic_vec<cuda::std::int64_t, simd::fixed_size<4>>;
|
||||
|
||||
__global__ void test_load_i64_4(const cuda::std::int64_t* in, cuda::std::int64_t* out)
|
||||
{
|
||||
Vec_i64_4 v = simd::unchecked_load<Vec_i64_4>(in, Vec_i64_4::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_i64_4::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXXX-LABEL: .visible .entry {{.*}}test_load_i64_2{{.*}}(
|
||||
; SMXXX: {{.*}}ld.global.{{(v4\.[bus]32|v2\.[bus]64)}}{{.*}}
|
||||
; SMXXX: {{.*}}st.global.{{(v4\.[bus]32|v2\.[bus]64)}}{{.*}}
|
||||
|
||||
; SM100-PLUS-LABEL: .visible .entry {{.*}}test_load_i64_4{{.*}}(
|
||||
; SM100-PLUS: {{.*}}ld.global.{{(v4\.[bus]64|v8\.[bus]32)}}{{.*}}
|
||||
; SM100-PLUS: {{.*}}st.global.{{(v4\.[bus]64|v8\.[bus]32)}}{{.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,51 @@
|
||||
#include <cuda/std/__simd/load.h>
|
||||
#include <cuda/std/__simd/store.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
// --- 8-byte tier: int8_t, N=8 ---
|
||||
|
||||
using Vec_i8_8 = simd::basic_vec<cuda::std::int8_t, simd::fixed_size<8>>;
|
||||
|
||||
__global__ void test_load_i8_8(const cuda::std::int8_t* in, cuda::std::int8_t* out)
|
||||
{
|
||||
Vec_i8_8 v = simd::unchecked_load<Vec_i8_8>(in, Vec_i8_8::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_i8_8::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
// --- 16-byte tier: int8_t, N=16 ---
|
||||
|
||||
using Vec_i8_16 = simd::basic_vec<cuda::std::int8_t, simd::fixed_size<16>>;
|
||||
|
||||
__global__ void test_load_i8_16(const cuda::std::int8_t* in, cuda::std::int8_t* out)
|
||||
{
|
||||
Vec_i8_16 v = simd::unchecked_load<Vec_i8_16>(in, Vec_i8_16::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_i8_16::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
// --- 32-byte tier: int8_t, N=32 ---
|
||||
|
||||
using Vec_i8_32 = simd::basic_vec<cuda::std::int8_t, simd::fixed_size<32>>;
|
||||
|
||||
__global__ void test_load_i8_32(const cuda::std::int8_t* in, cuda::std::int8_t* out)
|
||||
{
|
||||
Vec_i8_32 v = simd::unchecked_load<Vec_i8_32>(in, Vec_i8_32::size(), simd::flag_aligned);
|
||||
simd::unchecked_store(v, out, Vec_i8_32::size(), simd::flag_aligned);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXXX-LABEL: .visible .entry {{.*}}test_load_i8_8{{.*}}(
|
||||
; SMXXX: {{.*}}ld.global.{{([bus]64|v4\.[bus]16|v2\.[bus]32)}}{{.*}}
|
||||
; SMXXX: {{.*}}st.global.{{([bus]64|v4\.[bus]16|v2\.[bus]32)}}{{.*}}
|
||||
|
||||
; SMXXX-LABEL: .visible .entry {{.*}}test_load_i8_16{{.*}}(
|
||||
; SMXXX: {{.*}}ld.global.{{(v4\.[bus]32|v2\.[bus]64)}}{{.*}}
|
||||
; SMXXX: {{.*}}st.global.{{(v4\.[bus]32|v2\.[bus]64)}}{{.*}}
|
||||
|
||||
; SM100-PLUS-LABEL: .visible .entry {{.*}}test_load_i8_32{{.*}}(
|
||||
; SM100-PLUS: {{.*}}ld.global.{{(v4\.[bus]64|v8\.[bus]32)}}{{.*}}
|
||||
; SM100-PLUS: {{.*}}st.global.{{(v4\.[bus]64|v8\.[bus]32)}}{{.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,45 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
#if _CCCL_HAS_NVBF16()
|
||||
|
||||
# include <cuda_bf16.h>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_bf16_x1 = simd::basic_vec<__nv_bfloat16, simd::fixed_size<1>>;
|
||||
|
||||
__device__ Vec_bf16_x1 test_min_bf16(Vec_bf16_x1 a, Vec_bf16_x1 b, Vec_bf16_x1 c)
|
||||
{
|
||||
return simd::fmin(simd::fmin(a, b), c);
|
||||
}
|
||||
|
||||
__device__ Vec_bf16_x1 test_max_bf16(Vec_bf16_x1 a, Vec_bf16_x1 b, Vec_bf16_x1 c)
|
||||
{
|
||||
return simd::fmax(simd::fmax(a, b), c);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_max_bf16.*}}
|
||||
; SM90: {{.*VHMNMX.BF16_V2.*!PT.*}}
|
||||
; SM100: {{.*VHMNMX.BF16_V2.*!PT.*}}
|
||||
; SM103: {{.*VHMNMX.BF16_V2.*!PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_min_bf16.*}}
|
||||
; SM90: {{.*VHMNMX.BF16_V2.*PT.*}}
|
||||
; SM100: {{.*VHMNMX.BF16_V2.*PT.*}}
|
||||
; SM103: {{.*VHMNMX.BF16_V2.*PT.*}}
|
||||
|
||||
*/
|
||||
|
||||
#endif // _CCCL_HAS_NVBF16()
|
||||
@@ -0,0 +1,45 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
#if _CCCL_HAS_NVFP16()
|
||||
|
||||
# include <cuda_fp16.h>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_f16_x1 = simd::basic_vec<__half, simd::fixed_size<1>>;
|
||||
|
||||
__device__ Vec_f16_x1 test_min_f16(Vec_f16_x1 a, Vec_f16_x1 b, Vec_f16_x1 c)
|
||||
{
|
||||
return simd::fmin(simd::fmin(a, b), c);
|
||||
}
|
||||
|
||||
__device__ Vec_f16_x1 test_max_f16(Vec_f16_x1 a, Vec_f16_x1 b, Vec_f16_x1 c)
|
||||
{
|
||||
return simd::fmax(simd::fmax(a, b), c);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_max_f16.*}}
|
||||
; SM90: {{.*VHMNMX .*!PT.*}}
|
||||
; SM100: {{.*VHMNMX .*!PT.*}}
|
||||
; SM103: {{.*VHMNMX .*!PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_min_f16.*}}
|
||||
; SM90: {{.*VHMNMX .*PT.*}}
|
||||
; SM100: {{.*VHMNMX .*PT.*}}
|
||||
; SM103: {{.*VHMNMX .*PT.*}}
|
||||
|
||||
*/
|
||||
|
||||
#endif // _CCCL_HAS_NVFP16()
|
||||
@@ -0,0 +1,103 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_s32_x1 = simd::basic_vec<cuda::std::int32_t, simd::fixed_size<1>>;
|
||||
using Vec_u32_x1 = simd::basic_vec<cuda::std::uint32_t, simd::fixed_size<1>>;
|
||||
using Vec_s16_x2 = simd::basic_vec<cuda::std::int16_t, simd::fixed_size<2>>;
|
||||
using Vec_u16_x2 = simd::basic_vec<cuda::std::uint16_t, simd::fixed_size<2>>;
|
||||
|
||||
__device__ Vec_u32_x1 test_min_u32(Vec_u32_x1 a, Vec_u32_x1 b, Vec_u32_x1 c)
|
||||
{
|
||||
return simd::min(simd::min(a, b), c);
|
||||
}
|
||||
|
||||
__device__ Vec_s32_x1 test_min_s32(Vec_s32_x1 a, Vec_s32_x1 b, Vec_s32_x1 c)
|
||||
{
|
||||
return simd::min(simd::min(a, b), c);
|
||||
}
|
||||
|
||||
__device__ Vec_u32_x1 test_max_u32(Vec_u32_x1 a, Vec_u32_x1 b, Vec_u32_x1 c)
|
||||
{
|
||||
return simd::max(simd::max(a, b), c);
|
||||
}
|
||||
|
||||
__device__ Vec_s32_x1 test_max_s32(Vec_s32_x1 a, Vec_s32_x1 b, Vec_s32_x1 c)
|
||||
{
|
||||
return simd::max(simd::max(a, b), c);
|
||||
}
|
||||
|
||||
__device__ Vec_u16_x2 test_min_u16_x2(Vec_u16_x2 a, Vec_u16_x2 b, Vec_u16_x2 c)
|
||||
{
|
||||
return simd::min(simd::min(a, b), c);
|
||||
}
|
||||
|
||||
__device__ Vec_s16_x2 test_min_s16_x2(Vec_s16_x2 a, Vec_s16_x2 b, Vec_s16_x2 c)
|
||||
{
|
||||
return simd::min(simd::min(a, b), c);
|
||||
}
|
||||
|
||||
__device__ Vec_u16_x2 test_max_u16_x2(Vec_u16_x2 a, Vec_u16_x2 b, Vec_u16_x2 c)
|
||||
{
|
||||
return simd::max(simd::max(a, b), c);
|
||||
}
|
||||
|
||||
__device__ Vec_s16_x2 test_max_s16_x2(Vec_s16_x2 a, Vec_s16_x2 b, Vec_s16_x2 c)
|
||||
{
|
||||
return simd::max(simd::max(a, b), c);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_max_s16_x2.*}}
|
||||
; SM90: {{.*VIMNMX3.S16x2.*!PT.*}}
|
||||
; SM100: {{.*VIMNMX3.S16x2.*!PT.*}}
|
||||
; SM103: {{.*VIMNMX3.S16x2.*!PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_max_u16_x2.*}}
|
||||
; SM90: {{.*VIMNMX3.U16x2.*!PT.*}}
|
||||
; SM100: {{.*VIMNMX3.U16x2.*!PT.*}}
|
||||
; SM103: {{.*VIMNMX3.U16x2.*!PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_min_s16_x2.*}}
|
||||
; SM90: {{.*VIMNMX3.S16x2.*PT.*}}
|
||||
; SM100: {{.*VIMNMX3.S16x2.*PT.*}}
|
||||
; SM103: {{.*VIMNMX3.S16x2.*PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_min_u16_x2.*}}
|
||||
; SM90: {{.*VIMNMX3.U16x2.*PT.*}}
|
||||
; SM100: {{.*VIMNMX3.U16x2.*PT.*}}
|
||||
; SM103: {{.*VIMNMX3.U16x2.*PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_max_s32.*}}
|
||||
; SM90: {{.*VIMNMX3 .*!PT.*}}
|
||||
; SM100: {{.*VIMNMX3 .*!PT.*}}
|
||||
; SM103: {{.*VIMNMX3 .*!PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_max_u32.*}}
|
||||
; SM90: {{.*VIMNMX3.U32.*!PT.*}}
|
||||
; SM100: {{.*VIMNMX3.U32.*!PT.*}}
|
||||
; SM103: {{.*VIMNMX3.U32.*!PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_min_s32.*}}
|
||||
; SM90: {{.*VIMNMX3 .*PT.*}}
|
||||
; SM100: {{.*VIMNMX3 .*PT.*}}
|
||||
; SM103: {{.*VIMNMX3 .*PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_min_u32.*}}
|
||||
; SM90: {{.*VIMNMX3.U32.*PT.*}}
|
||||
; SM100: {{.*VIMNMX3.U32.*PT.*}}
|
||||
; SM103: {{.*VIMNMX3.U32.*PT.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,56 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_s16_x2 = simd::basic_vec<cuda::std::int16_t, simd::fixed_size<2>>;
|
||||
using Vec_u16_x2 = simd::basic_vec<cuda::std::uint16_t, simd::fixed_size<2>>;
|
||||
|
||||
__device__ Vec_u16_x2 test_min_u16_x2(Vec_u16_x2 lhs, Vec_u16_x2 rhs)
|
||||
{
|
||||
return simd::min(lhs, rhs);
|
||||
}
|
||||
|
||||
__device__ Vec_s16_x2 test_min_s16_x2(Vec_s16_x2 lhs, Vec_s16_x2 rhs)
|
||||
{
|
||||
return simd::min(lhs, rhs);
|
||||
}
|
||||
|
||||
__device__ Vec_u16_x2 test_max_u16_x2(Vec_u16_x2 lhs, Vec_u16_x2 rhs)
|
||||
{
|
||||
return simd::max(lhs, rhs);
|
||||
}
|
||||
|
||||
__device__ Vec_s16_x2 test_max_s16_x2(Vec_s16_x2 lhs, Vec_s16_x2 rhs)
|
||||
{
|
||||
return simd::max(lhs, rhs);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_max_s16_x2.*}}
|
||||
; SM90: {{.*VIMNMX.S16x2.*!PT.*}}
|
||||
; SM1XX: {{.*VIMNMX.S16x2.*!PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_max_u16_x2.*}}
|
||||
; SM90: {{.*VIMNMX.U16x2.*!PT.*}}
|
||||
; SM1XX: {{.*VIMNMX.U16x2.*!PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_min_s16_x2.*}}
|
||||
; SM90: {{.*VIMNMX.S16x2.*PT.*}}
|
||||
; SM1XX: {{.*VIMNMX.S16x2.*PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_min_u16_x2.*}}
|
||||
; SM90: {{.*VIMNMX.U16x2.*PT.*}}
|
||||
; SM1XX: {{.*VIMNMX.U16x2.*PT.*}}
|
||||
|
||||
*/
|
||||
@@ -0,0 +1,52 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++ in the CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/__simd_> // IWYU pragma: keep
|
||||
|
||||
namespace simd = cuda::std::simd;
|
||||
|
||||
using Vec_s8_x4 = simd::basic_vec<cuda::std::int8_t, simd::fixed_size<4>>;
|
||||
using Vec_u8_x4 = simd::basic_vec<cuda::std::uint8_t, simd::fixed_size<4>>;
|
||||
|
||||
__device__ Vec_u8_x4 test_min_u8_x4(Vec_u8_x4 lhs, Vec_u8_x4 rhs)
|
||||
{
|
||||
return simd::min(lhs, rhs);
|
||||
}
|
||||
|
||||
__device__ Vec_s8_x4 test_min_s8_x4(Vec_s8_x4 lhs, Vec_s8_x4 rhs)
|
||||
{
|
||||
return simd::min(lhs, rhs);
|
||||
}
|
||||
|
||||
__device__ Vec_u8_x4 test_max_u8_x4(Vec_u8_x4 lhs, Vec_u8_x4 rhs)
|
||||
{
|
||||
return simd::max(lhs, rhs);
|
||||
}
|
||||
|
||||
__device__ Vec_s8_x4 test_max_s8_x4(Vec_s8_x4 lhs, Vec_s8_x4 rhs)
|
||||
{
|
||||
return simd::max(lhs, rhs);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_max_s8_x4.*}}
|
||||
; SM120f: {{.*VIMNMX.S8x4.*!PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_max_u8_x4.*}}
|
||||
; SM120f: {{.*VIMNMX.U8x4.*!PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_min_s8_x4.*}}
|
||||
; SM120f: {{.*VIMNMX.S8x4.*PT.*}}
|
||||
|
||||
; SMXX-LABEL: {{[[:space:]]*}}Function : {{.*test_min_u8_x4.*}}
|
||||
; SM120f: {{.*VIMNMX.U8x4.*PT.*}}
|
||||
|
||||
*/
|
||||
Reference in New Issue
Block a user