[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
EngineX CI
@@ -0,0 +1,43 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_BARRIER_CLUSTER_H_
|
||||
#define _CUDA_PTX_BARRIER_CLUSTER_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 9.7.12.3. Parallel Synchronization and Communication Instructions: barrier.cluster
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster
|
||||
#include <cuda/__ptx/instructions/generated/barrier_cluster.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_BARRIER_CLUSTER_H_
|
||||
@@ -0,0 +1,44 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_BFIND_H_
|
||||
#define _CUDA_PTX_BFIND_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/__type_traits/is_integral.h>
|
||||
#include <cuda/std/__type_traits/is_signed.h>
|
||||
#include <cuda/std/__type_traits/is_unsigned.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/bfind.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_BFIND_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_BMSK_H_
|
||||
#define _CUDA_PTX_BMSK_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/bmsk.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_BMSK_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_CLUSTERLAUNCHCONTROL_H_
|
||||
#define _CUDA_PTX_CLUSTERLAUNCHCONTROL_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/clusterlaunchcontrol.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_CLUSTERLAUNCHCONTROL_H_
|
||||
@@ -0,0 +1,44 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_CP_ASYNC_BULK_H_
|
||||
#define _CUDA_PTX_CP_ASYNC_BULK_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 9.7.8.24.6. Data Movement and Conversion Instructions: cp.async.bulk
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk
|
||||
#include <cuda/__ptx/instructions/generated/cp_async_bulk.h>
|
||||
#include <cuda/__ptx/instructions/generated/cp_async_bulk_multicast.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_CP_ASYNC_BULK_H_
|
||||
@@ -0,0 +1,43 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_CP_ASYNC_BULK_COMMIT_GROUP_H_
|
||||
#define _CUDA_PTX_CP_ASYNC_BULK_COMMIT_GROUP_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 9.7.8.24.12. Data Movement and Conversion Instructions: cp.async.bulk.commit_group
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-commit-group
|
||||
#include <cuda/__ptx/instructions/generated/cp_async_bulk_commit_group.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_CP_ASYNC_BULK_COMMIT_GROUP_H_
|
||||
@@ -0,0 +1,45 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_CP_ASYNC_BULK_TENSOR_H_
|
||||
#define _CUDA_PTX_CP_ASYNC_BULK_TENSOR_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 9.7.8.24.9. Data Movement and Conversion Instructions: cp.async.bulk.tensor
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor
|
||||
#include <cuda/__ptx/instructions/generated/cp_async_bulk_tensor.h>
|
||||
#include <cuda/__ptx/instructions/generated/cp_async_bulk_tensor_gather_scatter.h>
|
||||
#include <cuda/__ptx/instructions/generated/cp_async_bulk_tensor_multicast.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_CP_ASYNC_BULK_TENSOR_H_
|
||||
@@ -0,0 +1,43 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_CP_ASYNC_BULK_WAIT_GROUP_H_
|
||||
#define _CUDA_PTX_CP_ASYNC_BULK_WAIT_GROUP_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 9.7.8.24.13. Data Movement and Conversion Instructions: cp.async.bulk.wait_group
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-wait-group
|
||||
#include <cuda/__ptx/instructions/generated/cp_async_bulk_wait_group.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_CP_ASYNC_BULK_WAIT_GROUP_H_
|
||||
@@ -0,0 +1,42 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_CP_ASYNC_MBARRIER_ARRIVE_H_
|
||||
#define _CUDA_PTX_CP_ASYNC_MBARRIER_ARRIVE_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/cp_async_mbarrier_arrive.h>
|
||||
#include <cuda/__ptx/instructions/generated/cp_async_mbarrier_arrive_noinc.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_CP_ASYNC_MBARRIER_ARRIVE_H_
|
||||
@@ -0,0 +1,60 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_CP_REDUCE_ASYNC_BULK_H_
|
||||
#define _CUDA_PTX_CP_REDUCE_ASYNC_BULK_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
// Forward-declare __half and __nv_bfloat16. The cuda_fp16.h and cuda_bf16.h are
|
||||
// expensive to include. The APIs use only pointers, so we do not have to define
|
||||
// the types. If the user wants to use these types, it is their responsibility
|
||||
// to include the headers.
|
||||
#if _LIBCUDACXX_HAS_NVFP16()
|
||||
struct __half;
|
||||
#endif // _LIBCUDACXX_HAS_NVFP16()
|
||||
#if _LIBCUDACXX_HAS_NVBF16()
|
||||
struct __nv_bfloat16;
|
||||
#endif // _LIBCUDACXX_HAS_NVBF16()
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 9.7.8.24.7. Data Movement and Conversion Instructions: cp.reduce.async.bulk
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk
|
||||
#include <cuda/__ptx/instructions/generated/cp_reduce_async_bulk.h>
|
||||
#if _LIBCUDACXX_HAS_NVFP16()
|
||||
# include <cuda/__ptx/instructions/generated/cp_reduce_async_bulk_f16.h>
|
||||
#endif // _LIBCUDACXX_HAS_NVFP16()
|
||||
#if _LIBCUDACXX_HAS_NVBF16()
|
||||
# include <cuda/__ptx/instructions/generated/cp_reduce_async_bulk_bf16.h>
|
||||
#endif // _LIBCUDACXX_HAS_NVBF16()
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_CP_REDUCE_ASYNC_BULK_H_
|
||||
@@ -0,0 +1,43 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_CP_REDUCE_ASYNC_BULK_TENSOR_H_
|
||||
#define _CUDA_PTX_CP_REDUCE_ASYNC_BULK_TENSOR_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 9.7.8.24.10. Data Movement and Conversion Instructions: cp.reduce.async.bulk.tensor
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor
|
||||
#include <cuda/__ptx/instructions/generated/cp_reduce_async_bulk_tensor.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_CP_REDUCE_ASYNC_BULK_TENSOR_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_ELECT_SYNC_H_
|
||||
#define _CUDA_PTX_ELECT_SYNC_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/elect_sync.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_ELECT_SYNC_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_EXIT_H_
|
||||
#define _CUDA_PTX_EXIT_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/exit.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_EXIT_H_
|
||||
@@ -0,0 +1,49 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_FENCE_H_
|
||||
#define _CUDA_PTX_FENCE_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 9.7.12.4. Parallel Synchronization and Communication Instructions: membar/fence
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar-fence
|
||||
#include <cuda/__ptx/instructions/generated/fence.h>
|
||||
#include <cuda/__ptx/instructions/generated/fence_mbarrier_init.h>
|
||||
#include <cuda/__ptx/instructions/generated/fence_proxy_alias.h>
|
||||
#include <cuda/__ptx/instructions/generated/fence_proxy_async.h>
|
||||
#include <cuda/__ptx/instructions/generated/fence_proxy_async_generic_sync_restrict.h>
|
||||
#include <cuda/__ptx/instructions/generated/fence_proxy_tensormap_generic.h>
|
||||
#include <cuda/__ptx/instructions/generated/fence_sync_restrict.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_FENCE_H_
|
||||
@@ -0,0 +1,115 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_BARRIER_CLUSTER_H_
|
||||
#define _CUDA_PTX_GENERATED_BARRIER_CLUSTER_H_
|
||||
|
||||
/*
|
||||
// barrier.cluster.arrive; // PTX ISA 78, SM_90
|
||||
// Marked volatile and as clobbering memory
|
||||
template <typename = void>
|
||||
__device__ static inline void barrier_cluster_arrive();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_barrier_cluster_arrive_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void barrier_cluster_arrive()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("barrier.cluster.arrive;" : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_barrier_cluster_arrive_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// barrier.cluster.wait; // PTX ISA 78, SM_90
|
||||
// Marked volatile and as clobbering memory
|
||||
template <typename = void>
|
||||
__device__ static inline void barrier_cluster_wait();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_barrier_cluster_wait_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void barrier_cluster_wait()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("barrier.cluster.wait;" : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_barrier_cluster_wait_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// barrier.cluster.arrive.sem; // PTX ISA 80, SM_90
|
||||
// .sem = { .release }
|
||||
// Marked volatile and as clobbering memory
|
||||
template <typename = void>
|
||||
__device__ static inline void barrier_cluster_arrive(
|
||||
cuda::ptx::sem_release_t);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_barrier_cluster_arrive_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void barrier_cluster_arrive(::cuda::ptx::sem_release_t)
|
||||
{
|
||||
// __sem == sem_release (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("barrier.cluster.arrive.release;" : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_barrier_cluster_arrive_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// barrier.cluster.arrive.sem; // PTX ISA 80, SM_90
|
||||
// .sem = { .relaxed }
|
||||
// Marked volatile
|
||||
template <typename = void>
|
||||
__device__ static inline void barrier_cluster_arrive(
|
||||
cuda::ptx::sem_relaxed_t);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_barrier_cluster_arrive_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void barrier_cluster_arrive(::cuda::ptx::sem_relaxed_t)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("barrier.cluster.arrive.relaxed;" : : :);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_barrier_cluster_arrive_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// barrier.cluster.wait.sem; // PTX ISA 80, SM_90
|
||||
// .sem = { .acquire }
|
||||
// Marked volatile and as clobbering memory
|
||||
template <typename = void>
|
||||
__device__ static inline void barrier_cluster_wait(
|
||||
cuda::ptx::sem_acquire_t);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_barrier_cluster_wait_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void barrier_cluster_wait(::cuda::ptx::sem_acquire_t)
|
||||
{
|
||||
// __sem == sem_acquire (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("barrier.cluster.wait.acquire;" : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_barrier_cluster_wait_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_BARRIER_CLUSTER_H_
|
||||
@@ -0,0 +1,154 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_BFIND_H_
|
||||
#define _CUDA_PTX_GENERATED_BFIND_H_
|
||||
|
||||
/*
|
||||
// bfind.u32 dest, a_reg; // PTX ISA 20, SM_50
|
||||
template <typename U32, enable_if_t<sizeof(U32) == 4 && is_integral_v<U32> && is_unsigned_v<U32>, bool> = true>
|
||||
__device__ static inline uint32_t bfind(
|
||||
U32 a_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <
|
||||
typename _U32,
|
||||
::cuda::std::enable_if_t<sizeof(_U32) == 4 && ::cuda::std::is_integral_v<_U32>&& ::cuda::std::is_unsigned_v<_U32>,
|
||||
bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind(_U32 __a_reg)
|
||||
{
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("bfind.u32 %0, %1;" : "=r"(__dest) : "r"(*reinterpret_cast<const ::cuda::std::uint32_t*>(&__a_reg)) :);
|
||||
return __dest;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// bfind.shiftamt.u32 dest, a_reg; // PTX ISA 20, SM_50
|
||||
template <typename U32, enable_if_t<sizeof(U32) == 4 && is_integral_v<U32> && is_unsigned_v<U32>, bool> = true>
|
||||
__device__ static inline uint32_t bfind_shiftamt(
|
||||
U32 a_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <
|
||||
typename _U32,
|
||||
::cuda::std::enable_if_t<sizeof(_U32) == 4 && ::cuda::std::is_integral_v<_U32>&& ::cuda::std::is_unsigned_v<_U32>,
|
||||
bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind_shiftamt(_U32 __a_reg)
|
||||
{
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("bfind.shiftamt.u32 %0, %1;" : "=r"(__dest) : "r"(*reinterpret_cast<const ::cuda::std::uint32_t*>(&__a_reg)) :);
|
||||
return __dest;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// bfind.u64 dest, a_reg; // PTX ISA 20, SM_50
|
||||
template <typename U64, enable_if_t<sizeof(U64) == 8 && is_integral_v<U64> && is_unsigned_v<U64>, bool> = true>
|
||||
__device__ static inline uint32_t bfind(
|
||||
U64 a_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <
|
||||
typename _U64,
|
||||
::cuda::std::enable_if_t<sizeof(_U64) == 8 && ::cuda::std::is_integral_v<_U64>&& ::cuda::std::is_unsigned_v<_U64>,
|
||||
bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind(_U64 __a_reg)
|
||||
{
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("bfind.u64 %0, %1;" : "=r"(__dest) : "l"(*reinterpret_cast<const ::cuda::std::uint64_t*>(&__a_reg)) :);
|
||||
return __dest;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// bfind.shiftamt.u64 dest, a_reg; // PTX ISA 20, SM_50
|
||||
template <typename U64, enable_if_t<sizeof(U64) == 8 && is_integral_v<U64> && is_unsigned_v<U64>, bool> = true>
|
||||
__device__ static inline uint32_t bfind_shiftamt(
|
||||
U64 a_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <
|
||||
typename _U64,
|
||||
::cuda::std::enable_if_t<sizeof(_U64) == 8 && ::cuda::std::is_integral_v<_U64>&& ::cuda::std::is_unsigned_v<_U64>,
|
||||
bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind_shiftamt(_U64 __a_reg)
|
||||
{
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("bfind.shiftamt.u64 %0, %1;" : "=r"(__dest) : "l"(*reinterpret_cast<const ::cuda::std::uint64_t*>(&__a_reg)) :);
|
||||
return __dest;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// bfind.s32 dest, a_reg; // PTX ISA 20, SM_50
|
||||
template <typename S32, enable_if_t<sizeof(S32) == 4 && is_integral_v<S32> && is_signed_v<S32>, bool> = true>
|
||||
__device__ static inline uint32_t bfind(
|
||||
S32 a_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename _S32,
|
||||
::cuda::std::enable_if_t<sizeof(_S32) == 4 && ::cuda::std::is_integral_v<_S32>&& ::cuda::std::is_signed_v<_S32>,
|
||||
bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind(_S32 __a_reg)
|
||||
{
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("bfind.s32 %0, %1;" : "=r"(__dest) : "r"(*reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)) :);
|
||||
return __dest;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// bfind.shiftamt.s32 dest, a_reg; // PTX ISA 20, SM_50
|
||||
template <typename S32, enable_if_t<sizeof(S32) == 4 && is_integral_v<S32> && is_signed_v<S32>, bool> = true>
|
||||
__device__ static inline uint32_t bfind_shiftamt(
|
||||
S32 a_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename _S32,
|
||||
::cuda::std::enable_if_t<sizeof(_S32) == 4 && ::cuda::std::is_integral_v<_S32>&& ::cuda::std::is_signed_v<_S32>,
|
||||
bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind_shiftamt(_S32 __a_reg)
|
||||
{
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("bfind.shiftamt.s32 %0, %1;" : "=r"(__dest) : "r"(*reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)) :);
|
||||
return __dest;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// bfind.s64 dest, a_reg; // PTX ISA 20, SM_50
|
||||
template <typename S64, enable_if_t<sizeof(S64) == 8 && is_integral_v<S64> && is_signed_v<S64>, bool> = true>
|
||||
__device__ static inline uint32_t bfind(
|
||||
S64 a_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename _S64,
|
||||
::cuda::std::enable_if_t<sizeof(_S64) == 8 && ::cuda::std::is_integral_v<_S64>&& ::cuda::std::is_signed_v<_S64>,
|
||||
bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind(_S64 __a_reg)
|
||||
{
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("bfind.s64 %0, %1;" : "=r"(__dest) : "l"(*reinterpret_cast<const ::cuda::std::int64_t*>(&__a_reg)) :);
|
||||
return __dest;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// bfind.shiftamt.s64 dest, a_reg; // PTX ISA 20, SM_50
|
||||
template <typename S64, enable_if_t<sizeof(S64) == 8 && is_integral_v<S64> && is_signed_v<S64>, bool> = true>
|
||||
__device__ static inline uint32_t bfind_shiftamt(
|
||||
S64 a_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename _S64,
|
||||
::cuda::std::enable_if_t<sizeof(_S64) == 8 && ::cuda::std::is_integral_v<_S64>&& ::cuda::std::is_signed_v<_S64>,
|
||||
bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t bfind_shiftamt(_S64 __a_reg)
|
||||
{
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("bfind.shiftamt.s64 %0, %1;" : "=r"(__dest) : "l"(*reinterpret_cast<const ::cuda::std::int64_t*>(&__a_reg)) :);
|
||||
return __dest;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_BFIND_H_
|
||||
@@ -0,0 +1,54 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_BMSK_H_
|
||||
#define _CUDA_PTX_GENERATED_BMSK_H_
|
||||
|
||||
/*
|
||||
// bmsk.clamp.b32 dest, a_reg, b_reg; // PTX ISA 76, SM_70
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t bmsk_clamp(
|
||||
uint32_t a_reg,
|
||||
uint32_t b_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 760
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_bmsk_clamp_is_not_supported_before_SM_70__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t bmsk_clamp(::cuda::std::uint32_t __a_reg, ::cuda::std::uint32_t __b_reg)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 700
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("bmsk.clamp.b32 %0, %1, %2;" : "=r"(__dest) : "r"(__a_reg), "r"(__b_reg) :);
|
||||
return __dest;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_bmsk_clamp_is_not_supported_before_SM_70__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 760
|
||||
|
||||
/*
|
||||
// bmsk.wrap.b32 dest, a_reg, b_reg; // PTX ISA 76, SM_70
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t bmsk_wrap(
|
||||
uint32_t a_reg,
|
||||
uint32_t b_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 760
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_bmsk_wrap_is_not_supported_before_SM_70__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t bmsk_wrap(::cuda::std::uint32_t __a_reg, ::cuda::std::uint32_t __b_reg)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 700
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("bmsk.wrap.b32 %0, %1, %2;" : "=r"(__dest) : "r"(__a_reg), "r"(__b_reg) :);
|
||||
return __dest;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_bmsk_wrap_is_not_supported_before_SM_70__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 760
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_BMSK_H_
|
||||
@@ -0,0 +1,240 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_CLUSTERLAUNCHCONTROL_H_
|
||||
#define _CUDA_PTX_GENERATED_CLUSTERLAUNCHCONTROL_H_
|
||||
|
||||
/*
|
||||
// clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [addr], [smem_bar]; // PTX ISA
|
||||
86, SM_100 template <typename = void>
|
||||
__device__ static inline void clusterlaunchcontrol_try_cancel(
|
||||
void* addr,
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_clusterlaunchcontrol_try_cancel_is_not_supported_before_SM_100__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void clusterlaunchcontrol_try_cancel(void* __addr, ::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
|
||||
asm("clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [%0], [%1];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_clusterlaunchcontrol_try_cancel_is_not_supported_before_SM_100__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.multicast::cluster::all.b128 [addr],
|
||||
[smem_bar]; // PTX ISA 86, SM_100a, SM_110a template <typename = void>
|
||||
__device__ static inline void clusterlaunchcontrol_try_cancel_multicast(
|
||||
void* addr,
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_clusterlaunchcontrol_try_cancel_multicast_is_only_supported_on_SM_100a_110a__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void
|
||||
clusterlaunchcontrol_try_cancel_multicast(void* __addr, ::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
|
||||
asm("clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.multicast::cluster::all.b128 "
|
||||
"[%0], [%1];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_clusterlaunchcontrol_try_cancel_multicast_is_only_supported_on_SM_100a_110a__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 pred_is_canceled, try_cancel_response; // PTX ISA 86, SM_100
|
||||
template <typename B128, enable_if_t<sizeof(B128) == 16, bool> = true>
|
||||
__device__ static inline bool clusterlaunchcontrol_query_cancel_is_canceled(
|
||||
B128 try_cancel_response);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_clusterlaunchcontrol_query_cancel_is_canceled_is_not_supported_before_SM_100__();
|
||||
template <typename _B128, ::cuda::std::enable_if_t<sizeof(_B128) == 16, bool> = true>
|
||||
_CCCL_DEVICE static inline bool clusterlaunchcontrol_query_cancel_is_canceled(_B128 __try_cancel_response)
|
||||
{
|
||||
static_assert(sizeof(_B128) == 16);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
|
||||
::cuda::std::uint32_t __pred_is_canceled;
|
||||
asm("{\n\t .reg .b128 B128_try_cancel_response; \n\t"
|
||||
"mov.b128 B128_try_cancel_response, {%1, %2}; \n"
|
||||
"{\n\t .reg .pred P_OUT; \n\t"
|
||||
"clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 P_OUT, B128_try_cancel_response;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}\n\t"
|
||||
"}"
|
||||
: "=r"(__pred_is_canceled)
|
||||
: "l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).x),
|
||||
"l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).y)
|
||||
:);
|
||||
return static_cast<bool>(__pred_is_canceled);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_clusterlaunchcontrol_query_cancel_is_canceled_is_not_supported_before_SM_100__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// clusterlaunchcontrol.query_cancel.get_first_ctaid::x.b32.b128 ret_dim, try_cancel_response; // PTX ISA 86, SM_100
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, enable_if_t<sizeof(B128) == 16, bool>
|
||||
= true>
|
||||
__device__ static inline B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_x(
|
||||
B128 try_cancel_response);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_x_is_not_supported_before_SM_100__();
|
||||
template <typename _B32,
|
||||
::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true,
|
||||
typename _B128,
|
||||
::cuda::std::enable_if_t<sizeof(_B128) == 16, bool> = true>
|
||||
_CCCL_DEVICE static inline _B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_x(_B128 __try_cancel_response)
|
||||
{
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
static_assert(sizeof(_B128) == 16);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
|
||||
::cuda::std::uint32_t __ret_dim;
|
||||
asm("{\n\t .reg .b128 B128_try_cancel_response; \n\t"
|
||||
"mov.b128 B128_try_cancel_response, {%1, %2}; \n"
|
||||
"clusterlaunchcontrol.query_cancel.get_first_ctaid::x.b32.b128 %0, B128_try_cancel_response;\n\t"
|
||||
"}"
|
||||
: "=r"(__ret_dim)
|
||||
: "l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).x),
|
||||
"l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).y)
|
||||
:);
|
||||
return *reinterpret_cast<_B32*>(&__ret_dim);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_x_is_not_supported_before_SM_100__();
|
||||
::cuda::std::uint32_t __err_out_var = 0;
|
||||
return *reinterpret_cast<_B32*>(&__err_out_var);
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// clusterlaunchcontrol.query_cancel.get_first_ctaid::y.b32.b128 ret_dim, try_cancel_response; // PTX ISA 86, SM_100
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, enable_if_t<sizeof(B128) == 16, bool>
|
||||
= true>
|
||||
__device__ static inline B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_y(
|
||||
B128 try_cancel_response);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_y_is_not_supported_before_SM_100__();
|
||||
template <typename _B32,
|
||||
::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true,
|
||||
typename _B128,
|
||||
::cuda::std::enable_if_t<sizeof(_B128) == 16, bool> = true>
|
||||
_CCCL_DEVICE static inline _B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_y(_B128 __try_cancel_response)
|
||||
{
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
static_assert(sizeof(_B128) == 16);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
|
||||
::cuda::std::uint32_t __ret_dim;
|
||||
asm("{\n\t .reg .b128 B128_try_cancel_response; \n\t"
|
||||
"mov.b128 B128_try_cancel_response, {%1, %2}; \n"
|
||||
"clusterlaunchcontrol.query_cancel.get_first_ctaid::y.b32.b128 %0, B128_try_cancel_response;\n\t"
|
||||
"}"
|
||||
: "=r"(__ret_dim)
|
||||
: "l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).x),
|
||||
"l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).y)
|
||||
:);
|
||||
return *reinterpret_cast<_B32*>(&__ret_dim);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_y_is_not_supported_before_SM_100__();
|
||||
::cuda::std::uint32_t __err_out_var = 0;
|
||||
return *reinterpret_cast<_B32*>(&__err_out_var);
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// clusterlaunchcontrol.query_cancel.get_first_ctaid::z.b32.b128 ret_dim, try_cancel_response; // PTX ISA 86, SM_100
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, enable_if_t<sizeof(B128) == 16, bool>
|
||||
= true>
|
||||
__device__ static inline B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_z(
|
||||
B128 try_cancel_response);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_z_is_not_supported_before_SM_100__();
|
||||
template <typename _B32,
|
||||
::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true,
|
||||
typename _B128,
|
||||
::cuda::std::enable_if_t<sizeof(_B128) == 16, bool> = true>
|
||||
_CCCL_DEVICE static inline _B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_z(_B128 __try_cancel_response)
|
||||
{
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
static_assert(sizeof(_B128) == 16);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
|
||||
::cuda::std::uint32_t __ret_dim;
|
||||
asm("{\n\t .reg .b128 B128_try_cancel_response; \n\t"
|
||||
"mov.b128 B128_try_cancel_response, {%1, %2}; \n"
|
||||
"clusterlaunchcontrol.query_cancel.get_first_ctaid::z.b32.b128 %0, B128_try_cancel_response;\n\t"
|
||||
"}"
|
||||
: "=r"(__ret_dim)
|
||||
: "l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).x),
|
||||
"l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).y)
|
||||
:);
|
||||
return *reinterpret_cast<_B32*>(&__ret_dim);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_z_is_not_supported_before_SM_100__();
|
||||
::cuda::std::uint32_t __err_out_var = 0;
|
||||
return *reinterpret_cast<_B32*>(&__err_out_var);
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 block_dim, try_cancel_response; // PTX ISA 86, SM_100
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, enable_if_t<sizeof(B128) == 16, bool>
|
||||
= true>
|
||||
__device__ static inline void clusterlaunchcontrol_query_cancel_get_first_ctaid(
|
||||
B32 (&block_dim)[4],
|
||||
B128 try_cancel_response);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_is_not_supported_before_SM_100__();
|
||||
template <typename _B32,
|
||||
::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true,
|
||||
typename _B128,
|
||||
::cuda::std::enable_if_t<sizeof(_B128) == 16, bool> = true>
|
||||
_CCCL_DEVICE static inline void
|
||||
clusterlaunchcontrol_query_cancel_get_first_ctaid(_B32 (&__block_dim)[4], _B128 __try_cancel_response)
|
||||
{
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
static_assert(sizeof(_B128) == 16);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
|
||||
asm("{\n\t .reg .b128 B128_try_cancel_response; \n\t"
|
||||
"mov.b128 B128_try_cancel_response, {%4, %5}; \n"
|
||||
"clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {%0, %1, %2, %3}, B128_try_cancel_response;\n\t"
|
||||
"}"
|
||||
: "=r"(__block_dim[0]), "=r"(__block_dim[1]), "=r"(__block_dim[2]), "=r"(__block_dim[3])
|
||||
: "l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).x),
|
||||
"l"((*reinterpret_cast<longlong2*>(&__try_cancel_response)).y)
|
||||
:);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_is_not_supported_before_SM_100__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_CLUSTERLAUNCHCONTROL_H_
|
||||
@@ -0,0 +1,245 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_CP_ASYNC_BULK_H_
|
||||
#define _CUDA_PTX_GENERATED_CP_ASYNC_BULK_H_
|
||||
|
||||
/*
|
||||
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes [dstMem], [srcMem], size, [smem_bar]; // PTX ISA 80, SM_90
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* srcMem,
|
||||
const uint32_t& size,
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __srcMem,
|
||||
const ::cuda::std::uint32_t& __size,
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)), "l"(__as_ptr_gmem(__srcMem)), "r"(__size), "r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes [dstMem], [srcMem], size, [smem_bar]; // PTX ISA 86, SM_90
|
||||
// .dst = { .shared::cta }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk(
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* srcMem,
|
||||
const uint32_t& size,
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk(
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __srcMem,
|
||||
const ::cuda::std::uint32_t& __size,
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)), "l"(__as_ptr_gmem(__srcMem)), "r"(__size), "r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes.ignore_oob [dstMem], [srcMem], size, ignoreBytesLeft,
|
||||
ignoreBytesRight, [smem_bar]; // PTX ISA 92, SM_90
|
||||
// .dst = { .shared::cta }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_ignore_oob(
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* srcMem,
|
||||
const uint32_t& size,
|
||||
const uint32_t& ignoreBytesLeft,
|
||||
const uint32_t& ignoreBytesRight,
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 920
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_ignore_oob_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_ignore_oob(
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __srcMem,
|
||||
const ::cuda::std::uint32_t& __size,
|
||||
const ::cuda::std::uint32_t& __ignoreBytesLeft,
|
||||
const ::cuda::std::uint32_t& __ignoreBytesRight,
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes.ignore_oob [%0], [%1], %2, %3, %4, [%5];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__as_ptr_gmem(__srcMem)),
|
||||
"r"(__size),
|
||||
"r"(__ignoreBytesLeft),
|
||||
"r"(__ignoreBytesRight),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_ignore_oob_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 920
|
||||
|
||||
/*
|
||||
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes [dstMem], [srcMem], size, [rdsmem_bar]; // PTX ISA 80, SM_90
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .shared::cta }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
void* dstMem,
|
||||
const void* srcMem,
|
||||
const uint32_t& size,
|
||||
uint64_t* rdsmem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
void* __dstMem,
|
||||
const void* __srcMem,
|
||||
const ::cuda::std::uint32_t& __size,
|
||||
::cuda::std::uint64_t* __rdsmem_bar)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];"
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__dstMem)),
|
||||
"r"(__as_ptr_smem(__srcMem)),
|
||||
"r"(__size),
|
||||
"r"(__as_ptr_remote_dsmem(__rdsmem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.dst.src.bulk_group [dstMem], [srcMem], size; // PTX ISA 80, SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
void* dstMem,
|
||||
const void* srcMem,
|
||||
const uint32_t& size);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
void* __dstMem,
|
||||
const void* __srcMem,
|
||||
const ::cuda::std::uint32_t& __size)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.global.shared::cta.bulk_group [%0], [%1], %2;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__dstMem)), "r"(__as_ptr_smem(__srcMem)), "r"(__size)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.dst.src.bulk_group.cp_mask [dstMem], [srcMem], size, byteMask; // PTX ISA 86, SM_100
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_cp_mask(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
void* dstMem,
|
||||
const void* srcMem,
|
||||
const uint32_t& size,
|
||||
const uint16_t& byteMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_cp_mask_is_not_supported_before_SM_100__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_cp_mask(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
void* __dstMem,
|
||||
const void* __srcMem,
|
||||
const ::cuda::std::uint32_t& __size,
|
||||
const ::cuda::std::uint16_t& __byteMask)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
|
||||
asm("cp.async.bulk.global.shared::cta.bulk_group.cp_mask [%0], [%1], %2, %3;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__dstMem)), "r"(__as_ptr_smem(__srcMem)), "r"(__size), "h"(__byteMask)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_cp_mask_is_not_supported_before_SM_100__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_CP_ASYNC_BULK_H_
|
||||
@@ -0,0 +1,25 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_CP_ASYNC_BULK_COMMIT_GROUP_H_
|
||||
#define _CUDA_PTX_GENERATED_CP_ASYNC_BULK_COMMIT_GROUP_H_
|
||||
|
||||
/*
|
||||
// cp.async.bulk.commit_group; // PTX ISA 80, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_commit_group();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_commit_group_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_commit_group()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("cp.async.bulk.commit_group;" : : :);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_commit_group_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_CP_ASYNC_BULK_COMMIT_GROUP_H_
|
||||
@@ -0,0 +1,52 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_CP_ASYNC_BULK_MULTICAST_H_
|
||||
#define _CUDA_PTX_GENERATED_CP_ASYNC_BULK_MULTICAST_H_
|
||||
|
||||
/*
|
||||
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [srcMem], size, [smem_bar], ctaMask;
|
||||
// PTX ISA 80, SM_90a, SM_100a, SM_110a
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* srcMem,
|
||||
const uint32_t& size,
|
||||
uint64_t* smem_bar,
|
||||
const uint16_t& ctaMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_is_only_supported_on_SM_90a_100a_110a__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __srcMem,
|
||||
const ::cuda::std::uint32_t& __size,
|
||||
::cuda::std::uint64_t* __smem_bar,
|
||||
const ::cuda::std::uint16_t& __ctaMask)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
|
||||
asm("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster [%0], [%1], %2, [%3], %4;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__as_ptr_gmem(__srcMem)),
|
||||
"r"(__size),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_is_only_supported_on_SM_90a_100a_110a__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_CP_ASYNC_BULK_MULTICAST_H_
|
||||
@@ -0,0 +1,977 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_CP_ASYNC_BULK_TENSOR_H_
|
||||
#define _CUDA_PTX_GENERATED_CP_ASYNC_BULK_TENSOR_H_
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.1d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; //
|
||||
PTX ISA 80, SM_90
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[1],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[1],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2}], [%3];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)), "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.1d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; //
|
||||
PTX ISA 86, SM_90
|
||||
// .dst = { .shared::cta }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[1],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[1],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.1d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2}], [%3];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)), "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.1d.dst.src.tile.mbarrier::complete_tx::bytes.cta_group [dstMem], [tensorMap, tensorCoords],
|
||||
[smem_bar]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cta }
|
||||
// .src = { .global }
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[1],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[1],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.1d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%0], [%1, {%2}], "
|
||||
"[%3];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)), "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.1d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%0], [%1, {%2}], "
|
||||
"[%3];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)), "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.1d.dst.src.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // PTX ISA 80, SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[1],
|
||||
const void* srcMem);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[1],
|
||||
const void* __srcMem)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.1d.global.shared::cta.tile.bulk_group [%0, {%1}], [%2];"
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.2d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; //
|
||||
PTX ISA 80, SM_90
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[2],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[2],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3}], [%4];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.2d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; //
|
||||
PTX ISA 86, SM_90
|
||||
// .dst = { .shared::cta }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[2],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[2],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.2d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3}], [%4];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.2d.dst.src.tile.mbarrier::complete_tx::bytes.cta_group [dstMem], [tensorMap, tensorCoords],
|
||||
[smem_bar]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cta }
|
||||
// .src = { .global }
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[2],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[2],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.2d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%0], [%1, {%2, "
|
||||
"%3}], [%4];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.2d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%0], [%1, {%2, "
|
||||
"%3}], [%4];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.2d.dst.src.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // PTX ISA 80, SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[2],
|
||||
const void* srcMem);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[2],
|
||||
const void* __srcMem)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group [%0, {%1, %2}], [%3];"
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__tensorCoords[1]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.3d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; //
|
||||
PTX ISA 80, SM_90
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[3],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[3],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4}], [%5];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.3d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; //
|
||||
PTX ISA 86, SM_90
|
||||
// .dst = { .shared::cta }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[3],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[3],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.3d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4}], [%5];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.3d.dst.src.tile.mbarrier::complete_tx::bytes.cta_group [dstMem], [tensorMap, tensorCoords],
|
||||
[smem_bar]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cta }
|
||||
// .src = { .global }
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[3],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[3],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.3d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%0], [%1, {%2, %3, "
|
||||
"%4}], [%5];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.3d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%0], [%1, {%2, %3, "
|
||||
"%4}], [%5];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.3d.dst.src.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // PTX ISA 80, SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[3],
|
||||
const void* srcMem);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[3],
|
||||
const void* __srcMem)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.3d.global.shared::cta.tile.bulk_group [%0, {%1, %2, %3}], [%4];"
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.4d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; //
|
||||
PTX ISA 80, SM_90
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[4],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[4],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4, %5}], "
|
||||
"[%6];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.4d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; //
|
||||
PTX ISA 86, SM_90
|
||||
// .dst = { .shared::cta }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[4],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[4],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.4d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4, %5}], [%6];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.4d.dst.src.tile.mbarrier::complete_tx::bytes.cta_group [dstMem], [tensorMap, tensorCoords],
|
||||
[smem_bar]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cta }
|
||||
// .src = { .global }
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[4],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[4],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.4d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%0], [%1, {%2, %3, "
|
||||
"%4, %5}], [%6];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.4d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%0], [%1, {%2, %3, "
|
||||
"%4, %5}], [%6];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.4d.dst.src.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // PTX ISA 80, SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[4],
|
||||
const void* srcMem);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[4],
|
||||
const void* __srcMem)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.4d.global.shared::cta.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5];"
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.5d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; //
|
||||
PTX ISA 80, SM_90
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[5],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[5],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4, %5, "
|
||||
"%6}], [%7];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.5d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; //
|
||||
PTX ISA 86, SM_90
|
||||
// .dst = { .shared::cta }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[5],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[5],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.5d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4, %5, %6}], "
|
||||
"[%7];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.5d.dst.src.tile.mbarrier::complete_tx::bytes.cta_group [dstMem], [tensorMap, tensorCoords],
|
||||
[smem_bar]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cta }
|
||||
// .src = { .global }
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[5],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[5],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.5d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%0], [%1, {%2, %3, "
|
||||
"%4, %5, %6}], [%7];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.5d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%0], [%1, {%2, %3, "
|
||||
"%4, %5, %6}], [%7];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.5d.dst.src.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // PTX ISA 80, SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[5],
|
||||
const void* srcMem);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[5],
|
||||
const void* __srcMem)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.async.bulk.tensor.5d.global.shared::cta.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6];"
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_CP_ASYNC_BULK_TENSOR_H_
|
||||
@@ -0,0 +1,302 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_CP_ASYNC_BULK_TENSOR_GATHER_SCATTER_H_
|
||||
#define _CUDA_PTX_GENERATED_CP_ASYNC_BULK_TENSOR_GATHER_SCATTER_H_
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.2d.dst.src.tile::gather4.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords],
|
||||
[smem_bar]; // PTX ISA 86, SM_100
|
||||
// .dst = { .shared::cta }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor_tile_gather4(
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[5],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_not_supported_before_SM_100__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor_tile_gather4(
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[5],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
|
||||
asm("cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4, "
|
||||
"%5, %6}], [%7];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_not_supported_before_SM_100__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.2d.dst.src.tile::gather4.mbarrier::complete_tx::bytes.cta_group [dstMem], [tensorMap,
|
||||
tensorCoords], [smem_bar]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cta }
|
||||
// .src = { .global }
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void cp_async_bulk_tensor_tile_gather4(
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[5],
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor_tile_gather4(
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[5],
|
||||
::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::1 [%0], [%1, "
|
||||
"{%2, %3, %4, %5, %6}], [%7];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::2 [%0], [%1, "
|
||||
"{%2, %3, %4, %5, %6}], [%7];"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__smem_bar))
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.2d.dst.src.tile::gather4.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [tensorMap,
|
||||
tensorCoords], [smem_bar], ctaMask; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor_tile_gather4(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[5],
|
||||
uint64_t* smem_bar,
|
||||
const uint16_t& ctaMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor_tile_gather4(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[5],
|
||||
::cuda::std::uint64_t* __smem_bar,
|
||||
const ::cuda::std::uint16_t& __ctaMask)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
asm("cp.async.bulk.tensor.2d.shared::cluster.global.tile::gather4.mbarrier::complete_tx::bytes.multicast::cluster "
|
||||
"[%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.2d.dst.src.tile::gather4.mbarrier::complete_tx::bytes.multicast::cluster.cta_group [dstMem],
|
||||
[tensorMap, tensorCoords], [smem_bar], ctaMask; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void cp_async_bulk_tensor_tile_gather4(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[5],
|
||||
uint64_t* smem_bar,
|
||||
const uint16_t& ctaMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor_tile_gather4(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[5],
|
||||
::cuda::std::uint64_t* __smem_bar,
|
||||
const ::cuda::std::uint16_t& __ctaMask)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.2d.shared::cluster.global.tile::gather4.mbarrier::complete_tx::bytes.multicast::cluster."
|
||||
"cta_group::1 [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.2d.shared::cluster.global.tile::gather4.mbarrier::complete_tx::bytes.multicast::cluster."
|
||||
"cta_group::2 [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.2d.dst.src.tile::scatter4.bulk_group [tensorMap, tensorCoords], [srcMem]; // PTX ISA 86,
|
||||
SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor_tile_scatter4(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[5],
|
||||
const void* srcMem);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_tile_scatter4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor_tile_scatter4(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[5],
|
||||
const void* __srcMem)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
asm("cp.async.bulk.tensor.2d.global.shared::cta.tile::scatter4.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6];"
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_tile_scatter4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_CP_ASYNC_BULK_TENSOR_GATHER_SCATTER_H_
|
||||
@@ -0,0 +1,631 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_CP_ASYNC_BULK_TENSOR_MULTICAST_H_
|
||||
#define _CUDA_PTX_GENERATED_CP_ASYNC_BULK_TENSOR_MULTICAST_H_
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.1d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [tensorMap,
|
||||
tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[1],
|
||||
uint64_t* smem_bar,
|
||||
const uint16_t& ctaMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[1],
|
||||
::cuda::std::uint64_t* __smem_bar,
|
||||
const ::cuda::std::uint16_t& __ctaMask)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
asm("cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%0], [%1, "
|
||||
"{%2}], [%3], %4;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.1d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group [dstMem], [tensorMap,
|
||||
tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[1],
|
||||
uint64_t* smem_bar,
|
||||
const uint16_t& ctaMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[1],
|
||||
::cuda::std::uint64_t* __smem_bar,
|
||||
const ::cuda::std::uint16_t& __ctaMask)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group:"
|
||||
":1 [%0], [%1, {%2}], [%3], %4;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group:"
|
||||
":2 [%0], [%1, {%2}], [%3], %4;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.2d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [tensorMap,
|
||||
tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[2],
|
||||
uint64_t* smem_bar,
|
||||
const uint16_t& ctaMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[2],
|
||||
::cuda::std::uint64_t* __smem_bar,
|
||||
const ::cuda::std::uint16_t& __ctaMask)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
asm("cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%0], [%1, "
|
||||
"{%2, %3}], [%4], %5;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.2d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group [dstMem], [tensorMap,
|
||||
tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[2],
|
||||
uint64_t* smem_bar,
|
||||
const uint16_t& ctaMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[2],
|
||||
::cuda::std::uint64_t* __smem_bar,
|
||||
const ::cuda::std::uint16_t& __ctaMask)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group:"
|
||||
":1 [%0], [%1, {%2, %3}], [%4], %5;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group:"
|
||||
":2 [%0], [%1, {%2, %3}], [%4], %5;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.3d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [tensorMap,
|
||||
tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[3],
|
||||
uint64_t* smem_bar,
|
||||
const uint16_t& ctaMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[3],
|
||||
::cuda::std::uint64_t* __smem_bar,
|
||||
const ::cuda::std::uint16_t& __ctaMask)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
asm("cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%0], [%1, "
|
||||
"{%2, %3, %4}], [%5], %6;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.3d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group [dstMem], [tensorMap,
|
||||
tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[3],
|
||||
uint64_t* smem_bar,
|
||||
const uint16_t& ctaMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[3],
|
||||
::cuda::std::uint64_t* __smem_bar,
|
||||
const ::cuda::std::uint16_t& __ctaMask)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group:"
|
||||
":1 [%0], [%1, {%2, %3, %4}], [%5], %6;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group:"
|
||||
":2 [%0], [%1, {%2, %3, %4}], [%5], %6;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.4d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [tensorMap,
|
||||
tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[4],
|
||||
uint64_t* smem_bar,
|
||||
const uint16_t& ctaMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[4],
|
||||
::cuda::std::uint64_t* __smem_bar,
|
||||
const ::cuda::std::uint16_t& __ctaMask)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
asm("cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%0], [%1, "
|
||||
"{%2, %3, %4, %5}], [%6], %7;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.4d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group [dstMem], [tensorMap,
|
||||
tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[4],
|
||||
uint64_t* smem_bar,
|
||||
const uint16_t& ctaMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[4],
|
||||
::cuda::std::uint64_t* __smem_bar,
|
||||
const ::cuda::std::uint16_t& __ctaMask)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group:"
|
||||
":1 [%0], [%1, {%2, %3, %4, %5}], [%6], %7;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group:"
|
||||
":2 [%0], [%1, {%2, %3, %4, %5}], [%6], %7;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.5d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [tensorMap,
|
||||
tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[5],
|
||||
uint64_t* smem_bar,
|
||||
const uint16_t& ctaMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[5],
|
||||
::cuda::std::uint64_t* __smem_bar,
|
||||
const ::cuda::std::uint16_t& __ctaMask)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
asm("cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%0], [%1, "
|
||||
"{%2, %3, %4, %5, %6}], [%7], %8;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.tensor.5d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group [dstMem], [tensorMap,
|
||||
tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .dst = { .shared::cluster }
|
||||
// .src = { .global }
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void cp_async_bulk_tensor(
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
void* dstMem,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[5],
|
||||
uint64_t* smem_bar,
|
||||
const uint16_t& ctaMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_tensor(
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group,
|
||||
void* __dstMem,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[5],
|
||||
::cuda::std::uint64_t* __smem_bar,
|
||||
const ::cuda::std::uint16_t& __ctaMask)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group:"
|
||||
":1 [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group:"
|
||||
":2 [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dstMem)),
|
||||
"l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__smem_bar)),
|
||||
"h"(__ctaMask)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_CP_ASYNC_BULK_TENSOR_MULTICAST_H_
|
||||
@@ -0,0 +1,46 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_CP_ASYNC_BULK_WAIT_GROUP_H_
|
||||
#define _CUDA_PTX_GENERATED_CP_ASYNC_BULK_WAIT_GROUP_H_
|
||||
|
||||
/*
|
||||
// cp.async.bulk.wait_group N; // PTX ISA 80, SM_90
|
||||
template <int N32>
|
||||
__device__ static inline void cp_async_bulk_wait_group(
|
||||
cuda::ptx::n32_t<N32> N);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_wait_group_is_not_supported_before_SM_90__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_wait_group(::cuda::ptx::n32_t<_N32> __N)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("cp.async.bulk.wait_group %0;" : : "n"(__N.value) : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_wait_group_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.async.bulk.wait_group.read N; // PTX ISA 80, SM_90
|
||||
template <int N32>
|
||||
__device__ static inline void cp_async_bulk_wait_group_read(
|
||||
cuda::ptx::n32_t<N32> N);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_bulk_wait_group_read_is_not_supported_before_SM_90__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void cp_async_bulk_wait_group_read(::cuda::ptx::n32_t<_N32> __N)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("cp.async.bulk.wait_group.read %0;" : : "n"(__N.value) : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_bulk_wait_group_read_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_CP_ASYNC_BULK_WAIT_GROUP_H_
|
||||
@@ -0,0 +1,26 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_CP_ASYNC_MBARRIER_ARRIVE_H_
|
||||
#define _CUDA_PTX_GENERATED_CP_ASYNC_MBARRIER_ARRIVE_H_
|
||||
|
||||
/*
|
||||
// cp.async.mbarrier.arrive.b64 [addr]; // PTX ISA 70, SM_80
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_mbarrier_arrive(
|
||||
uint64_t* addr);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 700
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_mbarrier_arrive_is_not_supported_before_SM_80__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_mbarrier_arrive(::cuda::std::uint64_t* __addr)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 800
|
||||
asm("cp.async.mbarrier.arrive.b64 [%0];" : : "r"(__as_ptr_smem(__addr)) : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_mbarrier_arrive_is_not_supported_before_SM_80__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 700
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_CP_ASYNC_MBARRIER_ARRIVE_H_
|
||||
@@ -0,0 +1,26 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_CP_ASYNC_MBARRIER_ARRIVE_NOINC_H_
|
||||
#define _CUDA_PTX_GENERATED_CP_ASYNC_MBARRIER_ARRIVE_NOINC_H_
|
||||
|
||||
/*
|
||||
// cp.async.mbarrier.arrive.noinc.b64 [addr]; // PTX ISA 70, SM_80
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_async_mbarrier_arrive_noinc(
|
||||
uint64_t* addr);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 700
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_async_mbarrier_arrive_noinc_is_not_supported_before_SM_80__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_async_mbarrier_arrive_noinc(::cuda::std::uint64_t* __addr)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 800
|
||||
asm("cp.async.mbarrier.arrive.noinc.b64 [%0];" : : "r"(__as_ptr_smem(__addr)) : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_async_mbarrier_arrive_noinc_is_not_supported_before_SM_80__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 700
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_CP_ASYNC_MBARRIER_ARRIVE_NOINC_H_
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,132 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_CP_REDUCE_ASYNC_BULK_BF16_H_
|
||||
#define _CUDA_PTX_GENERATED_CP_REDUCE_ASYNC_BULK_BF16_H_
|
||||
|
||||
/*
|
||||
// cp.reduce.async.bulk.dst.src.bulk_group.op.type [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
// .type = { .bf16 }
|
||||
// .op = { .min }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_reduce_async_bulk(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::op_min_t,
|
||||
__nv_bfloat16* dstMem,
|
||||
const __nv_bfloat16* srcMem,
|
||||
uint32_t size);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_reduce_async_bulk(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::op_min_t,
|
||||
__nv_bfloat16* __dstMem,
|
||||
const __nv_bfloat16* __srcMem,
|
||||
::cuda::std::uint32_t __size)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __type == type_bf16 (due to parameter type constraint)
|
||||
// __op == op_min (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.reduce.async.bulk.global.shared::cta.bulk_group.min.bf16 [%0], [%1], %2; // 4."
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__dstMem)), "r"(__as_ptr_smem(__srcMem)), "r"(__size)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.reduce.async.bulk.dst.src.bulk_group.op.type [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
// .type = { .bf16 }
|
||||
// .op = { .max }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_reduce_async_bulk(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::op_max_t,
|
||||
__nv_bfloat16* dstMem,
|
||||
const __nv_bfloat16* srcMem,
|
||||
uint32_t size);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_reduce_async_bulk(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::op_max_t,
|
||||
__nv_bfloat16* __dstMem,
|
||||
const __nv_bfloat16* __srcMem,
|
||||
::cuda::std::uint32_t __size)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __type == type_bf16 (due to parameter type constraint)
|
||||
// __op == op_max (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.reduce.async.bulk.global.shared::cta.bulk_group.max.bf16 [%0], [%1], %2; // 4."
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__dstMem)), "r"(__as_ptr_smem(__srcMem)), "r"(__size)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.reduce.async.bulk.dst.src.bulk_group.op.noftz.type [dstMem], [srcMem], size; // 5. PTX ISA 80, SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
// .type = { .bf16 }
|
||||
// .op = { .add }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_reduce_async_bulk(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::op_add_t,
|
||||
__nv_bfloat16* dstMem,
|
||||
const __nv_bfloat16* srcMem,
|
||||
uint32_t size);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_reduce_async_bulk(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::op_add_t,
|
||||
__nv_bfloat16* __dstMem,
|
||||
const __nv_bfloat16* __srcMem,
|
||||
::cuda::std::uint32_t __size)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __type == type_bf16 (due to parameter type constraint)
|
||||
// __op == op_add (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.reduce.async.bulk.global.shared::cta.bulk_group.add.noftz.bf16 [%0], [%1], %2; // 5."
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__dstMem)), "r"(__as_ptr_smem(__srcMem)), "r"(__size)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_CP_REDUCE_ASYNC_BULK_BF16_H_
|
||||
@@ -0,0 +1,132 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_CP_REDUCE_ASYNC_BULK_F16_H_
|
||||
#define _CUDA_PTX_GENERATED_CP_REDUCE_ASYNC_BULK_F16_H_
|
||||
|
||||
/*
|
||||
// cp.reduce.async.bulk.dst.src.bulk_group.op.type [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
// .type = { .f16 }
|
||||
// .op = { .min }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_reduce_async_bulk(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::op_min_t,
|
||||
__half* dstMem,
|
||||
const __half* srcMem,
|
||||
uint32_t size);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_reduce_async_bulk(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::op_min_t,
|
||||
__half* __dstMem,
|
||||
const __half* __srcMem,
|
||||
::cuda::std::uint32_t __size)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __type == type_f16 (due to parameter type constraint)
|
||||
// __op == op_min (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.reduce.async.bulk.global.shared::cta.bulk_group.min.f16 [%0], [%1], %2; // 4."
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__dstMem)), "r"(__as_ptr_smem(__srcMem)), "r"(__size)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.reduce.async.bulk.dst.src.bulk_group.op.type [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
// .type = { .f16 }
|
||||
// .op = { .max }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_reduce_async_bulk(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::op_max_t,
|
||||
__half* dstMem,
|
||||
const __half* srcMem,
|
||||
uint32_t size);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_reduce_async_bulk(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::op_max_t,
|
||||
__half* __dstMem,
|
||||
const __half* __srcMem,
|
||||
::cuda::std::uint32_t __size)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __type == type_f16 (due to parameter type constraint)
|
||||
// __op == op_max (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.reduce.async.bulk.global.shared::cta.bulk_group.max.f16 [%0], [%1], %2; // 4."
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__dstMem)), "r"(__as_ptr_smem(__srcMem)), "r"(__size)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.reduce.async.bulk.dst.src.bulk_group.op.noftz.type [dstMem], [srcMem], size; // 5. PTX ISA 80, SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
// .type = { .f16 }
|
||||
// .op = { .add }
|
||||
template <typename = void>
|
||||
__device__ static inline void cp_reduce_async_bulk(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::op_add_t,
|
||||
__half* dstMem,
|
||||
const __half* srcMem,
|
||||
uint32_t size);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void cp_reduce_async_bulk(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::op_add_t,
|
||||
__half* __dstMem,
|
||||
const __half* __srcMem,
|
||||
::cuda::std::uint32_t __size)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __type == type_f16 (due to parameter type constraint)
|
||||
// __op == op_add (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("cp.reduce.async.bulk.global.shared::cta.bulk_group.add.noftz.f16 [%0], [%1], %2; // 5."
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__dstMem)), "r"(__as_ptr_smem(__srcMem)), "r"(__size)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_CP_REDUCE_ASYNC_BULK_F16_H_
|
||||
@@ -0,0 +1,596 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_CP_REDUCE_ASYNC_BULK_TENSOR_H_
|
||||
#define _CUDA_PTX_GENERATED_CP_REDUCE_ASYNC_BULK_TENSOR_H_
|
||||
|
||||
/*
|
||||
// cp.reduce.async.bulk.tensor.1d.dst.src.op.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // 1a. PTX ISA 80,
|
||||
SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
// .op = { .add, .min, .max, .inc, .dec, .and, .or, .xor }
|
||||
template <cuda::ptx::dot_op Op>
|
||||
__device__ static inline void cp_reduce_async_bulk_tensor(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::op_t<Op> op,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[1],
|
||||
const void* srcMem);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_op _Op>
|
||||
_CCCL_DEVICE static inline void cp_reduce_async_bulk_tensor(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::op_t<_Op> __op,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[1],
|
||||
const void* __srcMem)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
static_assert(__op == op_add || __op == op_min || __op == op_max || __op == op_inc || __op == op_dec
|
||||
|| __op == op_and_op || __op == op_or_op || __op == op_xor_op);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__op == op_add)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.1d.global.shared::cta.add.tile.bulk_group [%0, {%1}], [%2]; // 1a."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_min)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.1d.global.shared::cta.min.tile.bulk_group [%0, {%1}], [%2]; // 1a."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_max)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.1d.global.shared::cta.max.tile.bulk_group [%0, {%1}], [%2]; // 1a."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_inc)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.1d.global.shared::cta.inc.tile.bulk_group [%0, {%1}], [%2]; // 1a."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_dec)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.1d.global.shared::cta.dec.tile.bulk_group [%0, {%1}], [%2]; // 1a."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_and_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.1d.global.shared::cta.and.tile.bulk_group [%0, {%1}], [%2]; // 1a."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_or_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.1d.global.shared::cta.or.tile.bulk_group [%0, {%1}], [%2]; // 1a."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_xor_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.1d.global.shared::cta.xor.tile.bulk_group [%0, {%1}], [%2]; // 1a."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.reduce.async.bulk.tensor.2d.dst.src.op.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // 1b. PTX ISA 80,
|
||||
SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
// .op = { .add, .min, .max, .inc, .dec, .and, .or, .xor }
|
||||
template <cuda::ptx::dot_op Op>
|
||||
__device__ static inline void cp_reduce_async_bulk_tensor(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::op_t<Op> op,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[2],
|
||||
const void* srcMem);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_op _Op>
|
||||
_CCCL_DEVICE static inline void cp_reduce_async_bulk_tensor(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::op_t<_Op> __op,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[2],
|
||||
const void* __srcMem)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
static_assert(__op == op_add || __op == op_min || __op == op_max || __op == op_inc || __op == op_dec
|
||||
|| __op == op_and_op || __op == op_or_op || __op == op_xor_op);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__op == op_add)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.2d.global.shared::cta.add.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__tensorCoords[1]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_min)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.2d.global.shared::cta.min.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__tensorCoords[1]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_max)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.2d.global.shared::cta.max.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__tensorCoords[1]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_inc)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.2d.global.shared::cta.inc.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__tensorCoords[1]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_dec)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.2d.global.shared::cta.dec.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__tensorCoords[1]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_and_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.2d.global.shared::cta.and.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__tensorCoords[1]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_or_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.2d.global.shared::cta.or.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__tensorCoords[1]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_xor_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.2d.global.shared::cta.xor.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
|
||||
:
|
||||
: "l"(__tensorMap), "r"(__tensorCoords[0]), "r"(__tensorCoords[1]), "r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.reduce.async.bulk.tensor.3d.dst.src.op.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // 1c. PTX ISA 80,
|
||||
SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
// .op = { .add, .min, .max, .inc, .dec, .and, .or, .xor }
|
||||
template <cuda::ptx::dot_op Op>
|
||||
__device__ static inline void cp_reduce_async_bulk_tensor(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::op_t<Op> op,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[3],
|
||||
const void* srcMem);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_op _Op>
|
||||
_CCCL_DEVICE static inline void cp_reduce_async_bulk_tensor(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::op_t<_Op> __op,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[3],
|
||||
const void* __srcMem)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
static_assert(__op == op_add || __op == op_min || __op == op_max || __op == op_inc || __op == op_dec
|
||||
|| __op == op_and_op || __op == op_or_op || __op == op_xor_op);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__op == op_add)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_min)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.3d.global.shared::cta.min.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_max)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.3d.global.shared::cta.max.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_inc)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.3d.global.shared::cta.inc.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_dec)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.3d.global.shared::cta.dec.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_and_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.3d.global.shared::cta.and.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_or_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.3d.global.shared::cta.or.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_xor_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.reduce.async.bulk.tensor.4d.dst.src.op.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // 1d. PTX ISA 80,
|
||||
SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
// .op = { .add, .min, .max, .inc, .dec, .and, .or, .xor }
|
||||
template <cuda::ptx::dot_op Op>
|
||||
__device__ static inline void cp_reduce_async_bulk_tensor(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::op_t<Op> op,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[4],
|
||||
const void* srcMem);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_op _Op>
|
||||
_CCCL_DEVICE static inline void cp_reduce_async_bulk_tensor(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::op_t<_Op> __op,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[4],
|
||||
const void* __srcMem)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
static_assert(__op == op_add || __op == op_min || __op == op_max || __op == op_inc || __op == op_dec
|
||||
|| __op == op_and_op || __op == op_or_op || __op == op_xor_op);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__op == op_add)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_min)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.4d.global.shared::cta.min.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_max)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.4d.global.shared::cta.max.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_inc)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.4d.global.shared::cta.inc.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_dec)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.4d.global.shared::cta.dec.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_and_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.4d.global.shared::cta.and.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_or_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.4d.global.shared::cta.or.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_xor_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.4d.global.shared::cta.xor.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// cp.reduce.async.bulk.tensor.5d.dst.src.op.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // 1e. PTX ISA 80,
|
||||
SM_90
|
||||
// .dst = { .global }
|
||||
// .src = { .shared::cta }
|
||||
// .op = { .add, .min, .max, .inc, .dec, .and, .or, .xor }
|
||||
template <cuda::ptx::dot_op Op>
|
||||
__device__ static inline void cp_reduce_async_bulk_tensor(
|
||||
cuda::ptx::space_global_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::op_t<Op> op,
|
||||
const void* tensorMap,
|
||||
const int32_t (&tensorCoords)[5],
|
||||
const void* srcMem);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_op _Op>
|
||||
_CCCL_DEVICE static inline void cp_reduce_async_bulk_tensor(
|
||||
::cuda::ptx::space_global_t,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::ptx::op_t<_Op> __op,
|
||||
const void* __tensorMap,
|
||||
const ::cuda::std::int32_t (&__tensorCoords)[5],
|
||||
const void* __srcMem)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
static_assert(__op == op_add || __op == op_min || __op == op_max || __op == op_inc || __op == op_dec
|
||||
|| __op == op_and_op || __op == op_or_op || __op == op_xor_op);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__op == op_add)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_min)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.5d.global.shared::cta.min.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_max)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.5d.global.shared::cta.max.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_inc)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.5d.global.shared::cta.inc.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_dec)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.5d.global.shared::cta.dec.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_and_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.5d.global.shared::cta.and.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_or_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.5d.global.shared::cta.or.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__op == op_xor_op)
|
||||
{
|
||||
asm("cp.reduce.async.bulk.tensor.5d.global.shared::cta.xor.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
|
||||
:
|
||||
: "l"(__tensorMap),
|
||||
"r"(__tensorCoords[0]),
|
||||
"r"(__tensorCoords[1]),
|
||||
"r"(__tensorCoords[2]),
|
||||
"r"(__tensorCoords[3]),
|
||||
"r"(__tensorCoords[4]),
|
||||
"r"(__as_ptr_smem(__srcMem))
|
||||
: "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_CP_REDUCE_ASYNC_BULK_TENSOR_H_
|
||||
@@ -0,0 +1,36 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_ELECT_SYNC_H_
|
||||
#define _CUDA_PTX_GENERATED_ELECT_SYNC_H_
|
||||
|
||||
/*
|
||||
// elect.sync _|is_elected, membermask; // PTX ISA 80, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline bool elect_sync(
|
||||
const uint32_t& membermask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_elect_sync_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline bool elect_sync(const ::cuda::std::uint32_t& __membermask)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __is_elected;
|
||||
asm volatile(
|
||||
"{\n\t .reg .pred P_OUT; \n\t"
|
||||
"elect.sync _|P_OUT, %1;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__is_elected)
|
||||
: "r"(__membermask)
|
||||
:);
|
||||
return static_cast<bool>(__is_elected);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_elect_sync_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_ELECT_SYNC_H_
|
||||
@@ -0,0 +1,25 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_EXIT_H_
|
||||
#define _CUDA_PTX_GENERATED_EXIT_H_
|
||||
|
||||
/*
|
||||
// exit; // PTX ISA 10, SM_50
|
||||
template <typename = void>
|
||||
__device__ static inline void exit();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 100
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_exit_is_not_supported_before_SM_50__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void exit()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
|
||||
asm volatile("exit;" : : :);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_exit_is_not_supported_before_SM_50__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 100
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_EXIT_H_
|
||||
@@ -0,0 +1,212 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_FENCE_H_
|
||||
#define _CUDA_PTX_GENERATED_FENCE_H_
|
||||
|
||||
/*
|
||||
// fence.sem.scope; // 1. PTX ISA 60, SM_70
|
||||
// .sem = { .sc }
|
||||
// .scope = { .cta, .gpu, .sys }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline void fence(
|
||||
cuda::ptx::sem_sc_t,
|
||||
cuda::ptx::scope_t<Scope> scope);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 600
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_is_not_supported_before_SM_70__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline void fence(::cuda::ptx::sem_sc_t, ::cuda::ptx::scope_t<_Scope> __scope)
|
||||
{
|
||||
// __sem == sem_sc (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_gpu || __scope == scope_sys);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 700
|
||||
// NOLINTBEGIN(bugprone-branch-clone)
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm volatile("fence.sc.cta; // 1." : : : "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_gpu)
|
||||
{
|
||||
asm volatile("fence.sc.gpu; // 1." : : : "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_sys)
|
||||
{
|
||||
asm volatile("fence.sc.sys; // 1." : : : "memory");
|
||||
}
|
||||
// NOLINTEND(bugprone-branch-clone)
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_is_not_supported_before_SM_70__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 600
|
||||
|
||||
/*
|
||||
// fence.sem.scope; // 2. PTX ISA 78, SM_90
|
||||
// .sem = { .sc }
|
||||
// .scope = { .cluster }
|
||||
template <typename = void>
|
||||
__device__ static inline void fence(
|
||||
cuda::ptx::sem_sc_t,
|
||||
cuda::ptx::scope_cluster_t);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void fence(::cuda::ptx::sem_sc_t, ::cuda::ptx::scope_cluster_t)
|
||||
{
|
||||
// __sem == sem_sc (due to parameter type constraint)
|
||||
// __scope == scope_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("fence.sc.cluster; // 2." : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// fence.sem.scope; // 1. PTX ISA 60, SM_70
|
||||
// .sem = { .acq_rel }
|
||||
// .scope = { .cta, .gpu, .sys }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline void fence(
|
||||
cuda::ptx::sem_acq_rel_t,
|
||||
cuda::ptx::scope_t<Scope> scope);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 600
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_is_not_supported_before_SM_70__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline void fence(::cuda::ptx::sem_acq_rel_t, ::cuda::ptx::scope_t<_Scope> __scope)
|
||||
{
|
||||
// __sem == sem_acq_rel (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_gpu || __scope == scope_sys);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 700
|
||||
// NOLINTBEGIN(bugprone-branch-clone)
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm volatile("fence.acq_rel.cta; // 1." : : : "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_gpu)
|
||||
{
|
||||
asm volatile("fence.acq_rel.gpu; // 1." : : : "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_sys)
|
||||
{
|
||||
asm volatile("fence.acq_rel.sys; // 1." : : : "memory");
|
||||
}
|
||||
// NOLINTEND(bugprone-branch-clone)
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_is_not_supported_before_SM_70__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 600
|
||||
|
||||
/*
|
||||
// fence.sem.scope; // 2. PTX ISA 78, SM_90
|
||||
// .sem = { .acq_rel }
|
||||
// .scope = { .cluster }
|
||||
template <typename = void>
|
||||
__device__ static inline void fence(
|
||||
cuda::ptx::sem_acq_rel_t,
|
||||
cuda::ptx::scope_cluster_t);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void fence(::cuda::ptx::sem_acq_rel_t, ::cuda::ptx::scope_cluster_t)
|
||||
{
|
||||
// __sem == sem_acq_rel (due to parameter type constraint)
|
||||
// __scope == scope_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("fence.acq_rel.cluster; // 2." : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// fence.sem.scope; // PTX ISA 86, SM_90
|
||||
// .sem = { .acquire }
|
||||
// .scope = { .cta, .cluster, .gpu, .sys }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline void fence(
|
||||
cuda::ptx::sem_acquire_t,
|
||||
cuda::ptx::scope_t<Scope> scope);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline void fence(::cuda::ptx::sem_acquire_t, ::cuda::ptx::scope_t<_Scope> __scope)
|
||||
{
|
||||
// __sem == sem_acquire (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster || __scope == scope_gpu || __scope == scope_sys);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm volatile("fence.acquire.cta;" : : : "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm volatile("fence.acquire.cluster;" : : : "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_gpu)
|
||||
{
|
||||
asm volatile("fence.acquire.gpu;" : : : "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_sys)
|
||||
{
|
||||
asm volatile("fence.acquire.sys;" : : : "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// fence.sem.scope; // PTX ISA 86, SM_90
|
||||
// .sem = { .release }
|
||||
// .scope = { .cta, .cluster, .gpu, .sys }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline void fence(
|
||||
cuda::ptx::sem_release_t,
|
||||
cuda::ptx::scope_t<Scope> scope);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline void fence(::cuda::ptx::sem_release_t, ::cuda::ptx::scope_t<_Scope> __scope)
|
||||
{
|
||||
// __sem == sem_release (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster || __scope == scope_gpu || __scope == scope_sys);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm volatile("fence.release.cta;" : : : "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm volatile("fence.release.cluster;" : : : "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_gpu)
|
||||
{
|
||||
asm volatile("fence.release.gpu;" : : : "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_sys)
|
||||
{
|
||||
asm volatile("fence.release.sys;" : : : "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_FENCE_H_
|
||||
@@ -0,0 +1,31 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_FENCE_MBARRIER_INIT_H_
|
||||
#define _CUDA_PTX_GENERATED_FENCE_MBARRIER_INIT_H_
|
||||
|
||||
/*
|
||||
// fence.mbarrier_init.sem.scope; // 3. PTX ISA 80, SM_90
|
||||
// .sem = { .release }
|
||||
// .scope = { .cluster }
|
||||
template <typename = void>
|
||||
__device__ static inline void fence_mbarrier_init(
|
||||
cuda::ptx::sem_release_t,
|
||||
cuda::ptx::scope_cluster_t);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_mbarrier_init_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void fence_mbarrier_init(::cuda::ptx::sem_release_t, ::cuda::ptx::scope_cluster_t)
|
||||
{
|
||||
// __sem == sem_release (due to parameter type constraint)
|
||||
// __scope == scope_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("fence.mbarrier_init.release.cluster; // 3." : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_mbarrier_init_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_FENCE_MBARRIER_INIT_H_
|
||||
@@ -0,0 +1,25 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_FENCE_PROXY_ALIAS_H_
|
||||
#define _CUDA_PTX_GENERATED_FENCE_PROXY_ALIAS_H_
|
||||
|
||||
/*
|
||||
// fence.proxy.alias; // 4. PTX ISA 75, SM_70
|
||||
template <typename = void>
|
||||
__device__ static inline void fence_proxy_alias();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 750
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_proxy_alias_is_not_supported_before_SM_70__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void fence_proxy_alias()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 700
|
||||
asm volatile("fence.proxy.alias; // 4." : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_proxy_alias_is_not_supported_before_SM_70__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 750
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_FENCE_PROXY_ALIAS_H_
|
||||
@@ -0,0 +1,58 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_FENCE_PROXY_ASYNC_H_
|
||||
#define _CUDA_PTX_GENERATED_FENCE_PROXY_ASYNC_H_
|
||||
|
||||
/*
|
||||
// fence.proxy.async; // 5. PTX ISA 80, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline void fence_proxy_async();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_proxy_async_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void fence_proxy_async()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("fence.proxy.async; // 5." : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_proxy_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// fence.proxy.async.space; // 6. PTX ISA 80, SM_90
|
||||
// .space = { .global, .shared::cluster, .shared::cta }
|
||||
template <cuda::ptx::dot_space Space>
|
||||
__device__ static inline void fence_proxy_async(
|
||||
cuda::ptx::space_t<Space> space);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_proxy_async_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_space _Space>
|
||||
_CCCL_DEVICE static inline void fence_proxy_async(::cuda::ptx::space_t<_Space> __space)
|
||||
{
|
||||
static_assert(__space == space_global || __space == space_cluster || __space == space_shared);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__space == space_global)
|
||||
{
|
||||
asm volatile("fence.proxy.async.global; // 6." : : : "memory");
|
||||
}
|
||||
else if constexpr (__space == space_cluster)
|
||||
{
|
||||
asm volatile("fence.proxy.async.shared::cluster; // 6." : : : "memory");
|
||||
}
|
||||
else if constexpr (__space == space_shared)
|
||||
{
|
||||
asm volatile("fence.proxy.async.shared::cta; // 6." : : : "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_proxy_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_FENCE_PROXY_ASYNC_H_
|
||||
@@ -0,0 +1,64 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_FENCE_PROXY_ASYNC_GENERIC_SYNC_RESTRICT_H_
|
||||
#define _CUDA_PTX_GENERATED_FENCE_PROXY_ASYNC_GENERIC_SYNC_RESTRICT_H_
|
||||
|
||||
/*
|
||||
// fence.proxy.async::generic.sem.sync_restrict::space.scope; // PTX ISA 86, SM_90
|
||||
// .sem = { .acquire }
|
||||
// .space = { .shared::cluster }
|
||||
// .scope = { .cluster }
|
||||
template <typename = void>
|
||||
__device__ static inline void fence_proxy_async_generic_sync_restrict(
|
||||
cuda::ptx::sem_acquire_t,
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::scope_cluster_t);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_proxy_async_generic_sync_restrict_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void fence_proxy_async_generic_sync_restrict(
|
||||
::cuda::ptx::sem_acquire_t, ::cuda::ptx::space_cluster_t, ::cuda::ptx::scope_cluster_t)
|
||||
{
|
||||
// __sem == sem_acquire (due to parameter type constraint)
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __scope == scope_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("fence.proxy.async::generic.acquire.sync_restrict::shared::cluster.cluster;" : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_proxy_async_generic_sync_restrict_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// fence.proxy.async::generic.sem.sync_restrict::space.scope; // PTX ISA 86, SM_90
|
||||
// .sem = { .release }
|
||||
// .space = { .shared::cta }
|
||||
// .scope = { .cluster }
|
||||
template <typename = void>
|
||||
__device__ static inline void fence_proxy_async_generic_sync_restrict(
|
||||
cuda::ptx::sem_release_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::scope_cluster_t);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_proxy_async_generic_sync_restrict_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void fence_proxy_async_generic_sync_restrict(
|
||||
::cuda::ptx::sem_release_t, ::cuda::ptx::space_shared_t, ::cuda::ptx::scope_cluster_t)
|
||||
{
|
||||
// __sem == sem_release (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __scope == scope_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("fence.proxy.async::generic.release.sync_restrict::shared::cta.cluster;" : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_proxy_async_generic_sync_restrict_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_FENCE_PROXY_ASYNC_GENERIC_SYNC_RESTRICT_H_
|
||||
@@ -0,0 +1,102 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_FENCE_PROXY_TENSORMAP_GENERIC_H_
|
||||
#define _CUDA_PTX_GENERATED_FENCE_PROXY_TENSORMAP_GENERIC_H_
|
||||
|
||||
/*
|
||||
// fence.proxy.tensormap::generic.release.scope; // 7. PTX ISA 83, SM_90
|
||||
// .sem = { .release }
|
||||
// .scope = { .cta, .cluster, .gpu, .sys }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline void fence_proxy_tensormap_generic(
|
||||
cuda::ptx::sem_release_t,
|
||||
cuda::ptx::scope_t<Scope> scope);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_proxy_tensormap_generic_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline void
|
||||
fence_proxy_tensormap_generic(::cuda::ptx::sem_release_t, ::cuda::ptx::scope_t<_Scope> __scope)
|
||||
{
|
||||
// __sem == sem_release (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster || __scope == scope_gpu || __scope == scope_sys);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm volatile("fence.proxy.tensormap::generic.release.cta; // 7." : : : "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm volatile("fence.proxy.tensormap::generic.release.cluster; // 7." : : : "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_gpu)
|
||||
{
|
||||
asm volatile("fence.proxy.tensormap::generic.release.gpu; // 7." : : : "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_sys)
|
||||
{
|
||||
asm volatile("fence.proxy.tensormap::generic.release.sys; // 7." : : : "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_proxy_tensormap_generic_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// fence.proxy.tensormap::generic.sem.scope [addr], size; // 8. PTX ISA 83, SM_90
|
||||
// .sem = { .acquire }
|
||||
// .scope = { .cta, .cluster, .gpu, .sys }
|
||||
template <int N32, cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline void fence_proxy_tensormap_generic(
|
||||
cuda::ptx::sem_acquire_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
const void* addr,
|
||||
cuda::ptx::n32_t<N32> size);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_proxy_tensormap_generic_is_not_supported_before_SM_90__();
|
||||
template <int _N32, ::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline void fence_proxy_tensormap_generic(
|
||||
::cuda::ptx::sem_acquire_t, ::cuda::ptx::scope_t<_Scope> __scope, const void* __addr, ::cuda::ptx::n32_t<_N32> __size)
|
||||
{
|
||||
// __sem == sem_acquire (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster || __scope == scope_gpu || __scope == scope_sys);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm volatile("fence.proxy.tensormap::generic.acquire.cta [%0], %1; // 8."
|
||||
:
|
||||
: "l"(__addr), "n"(__size.value)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm volatile("fence.proxy.tensormap::generic.acquire.cluster [%0], %1; // 8."
|
||||
:
|
||||
: "l"(__addr), "n"(__size.value)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_gpu)
|
||||
{
|
||||
asm volatile("fence.proxy.tensormap::generic.acquire.gpu [%0], %1; // 8."
|
||||
:
|
||||
: "l"(__addr), "n"(__size.value)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_sys)
|
||||
{
|
||||
asm volatile("fence.proxy.tensormap::generic.acquire.sys [%0], %1; // 8."
|
||||
:
|
||||
: "l"(__addr), "n"(__size.value)
|
||||
: "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_proxy_tensormap_generic_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_FENCE_PROXY_TENSORMAP_GENERIC_H_
|
||||
@@ -0,0 +1,64 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_FENCE_SYNC_RESTRICT_H_
|
||||
#define _CUDA_PTX_GENERATED_FENCE_SYNC_RESTRICT_H_
|
||||
|
||||
/*
|
||||
// fence.sem.sync_restrict::space.scope; // PTX ISA 86, SM_90
|
||||
// .sem = { .acquire }
|
||||
// .space = { .shared::cluster }
|
||||
// .scope = { .cluster }
|
||||
template <typename = void>
|
||||
__device__ static inline void fence_sync_restrict(
|
||||
cuda::ptx::sem_acquire_t,
|
||||
cuda::ptx::space_cluster_t,
|
||||
cuda::ptx::scope_cluster_t);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_sync_restrict_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void
|
||||
fence_sync_restrict(::cuda::ptx::sem_acquire_t, ::cuda::ptx::space_cluster_t, ::cuda::ptx::scope_cluster_t)
|
||||
{
|
||||
// __sem == sem_acquire (due to parameter type constraint)
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
// __scope == scope_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("fence.acquire.sync_restrict::shared::cluster.cluster;" : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_sync_restrict_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// fence.sem.sync_restrict::space.scope; // PTX ISA 86, SM_90
|
||||
// .sem = { .release }
|
||||
// .space = { .shared::cta }
|
||||
// .scope = { .cluster }
|
||||
template <typename = void>
|
||||
__device__ static inline void fence_sync_restrict(
|
||||
cuda::ptx::sem_release_t,
|
||||
cuda::ptx::space_shared_t,
|
||||
cuda::ptx::scope_cluster_t);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_fence_sync_restrict_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void
|
||||
fence_sync_restrict(::cuda::ptx::sem_release_t, ::cuda::ptx::space_shared_t, ::cuda::ptx::scope_cluster_t)
|
||||
{
|
||||
// __sem == sem_release (due to parameter type constraint)
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
// __scope == scope_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm volatile("fence.release.sync_restrict::shared::cta.cluster;" : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_fence_sync_restrict_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_FENCE_SYNC_RESTRICT_H_
|
||||
@@ -0,0 +1,949 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_GET_SREG_H_
|
||||
#define _CUDA_PTX_GENERATED_GET_SREG_H_
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%tid.x; // PTX ISA 20
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_tid_x();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_tid_x()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%tid.x;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%tid.y; // PTX ISA 20
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_tid_y();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_tid_y()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%tid.y;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%tid.z; // PTX ISA 20
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_tid_z();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_tid_z()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%tid.z;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%ntid.x; // PTX ISA 20
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_ntid_x();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ntid_x()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm volatile("mov.u32 %0, %%ntid.x;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%ntid.y; // PTX ISA 20
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_ntid_y();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ntid_y()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm volatile("mov.u32 %0, %%ntid.y;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%ntid.z; // PTX ISA 20
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_ntid_z();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ntid_z()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm volatile("mov.u32 %0, %%ntid.z;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%laneid; // PTX ISA 13
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_laneid();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 130
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_laneid()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%laneid;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 130
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%warpid; // PTX ISA 13
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_warpid();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 130
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_warpid()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm volatile("mov.u32 %0, %%warpid;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 130
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%nwarpid; // PTX ISA 20, SM_35
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_nwarpid();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nwarpid_is_not_supported_before_SM_35__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nwarpid()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm volatile("mov.u32 %0, %%nwarpid;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_nwarpid_is_not_supported_before_SM_35__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%ctaid.x; // PTX ISA 20
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_ctaid_x();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ctaid_x()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%ctaid.x;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%ctaid.y; // PTX ISA 20
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_ctaid_y();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ctaid_y()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%ctaid.y;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%ctaid.z; // PTX ISA 20
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_ctaid_z();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ctaid_z()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%ctaid.z;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%nctaid.x; // PTX ISA 20
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_nctaid_x();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nctaid_x()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%nctaid.x;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%nctaid.y; // PTX ISA 20
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_nctaid_y();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nctaid_y()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%nctaid.y;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%nctaid.z; // PTX ISA 20
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_nctaid_z();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nctaid_z()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%nctaid.z;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%smid; // PTX ISA 13
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_smid();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 130
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_smid()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%smid;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 130
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%nsmid; // PTX ISA 20, SM_35
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_nsmid();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nsmid_is_not_supported_before_SM_35__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nsmid()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm volatile("mov.u32 %0, %%nsmid;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_nsmid_is_not_supported_before_SM_35__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u64 sreg_value, %%gridid; // PTX ISA 30
|
||||
template <typename = void>
|
||||
__device__ static inline uint64_t get_sreg_gridid();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 300
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint64_t get_sreg_gridid()
|
||||
{
|
||||
::cuda::std::uint64_t __sreg_value;
|
||||
asm("mov.u64 %0, %%gridid;" : "=l"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 300
|
||||
|
||||
/*
|
||||
// mov.pred sreg_value, %%is_explicit_cluster; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline bool get_sreg_is_explicit_cluster();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_is_explicit_cluster_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline bool get_sreg_is_explicit_cluster()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mov.pred P_OUT, %%is_explicit_cluster;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__sreg_value)
|
||||
:
|
||||
:);
|
||||
return static_cast<bool>(__sreg_value);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_is_explicit_cluster_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%clusterid.x; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_clusterid_x();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clusterid_x_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clusterid_x()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%clusterid.x;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_clusterid_x_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%clusterid.y; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_clusterid_y();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clusterid_y_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clusterid_y()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%clusterid.y;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_clusterid_y_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%clusterid.z; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_clusterid_z();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clusterid_z_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clusterid_z()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%clusterid.z;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_clusterid_z_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%nclusterid.x; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_nclusterid_x();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nclusterid_x_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nclusterid_x()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%nclusterid.x;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_nclusterid_x_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%nclusterid.y; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_nclusterid_y();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nclusterid_y_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nclusterid_y()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%nclusterid.y;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_nclusterid_y_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%nclusterid.z; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_nclusterid_z();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nclusterid_z_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nclusterid_z()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%nclusterid.z;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_nclusterid_z_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%cluster_ctaid.x; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_cluster_ctaid_x();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_ctaid_x_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_ctaid_x()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%cluster_ctaid.x;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_cluster_ctaid_x_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%cluster_ctaid.y; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_cluster_ctaid_y();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_ctaid_y_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_ctaid_y()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%cluster_ctaid.y;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_cluster_ctaid_y_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%cluster_ctaid.z; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_cluster_ctaid_z();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_ctaid_z_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_ctaid_z()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%cluster_ctaid.z;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_cluster_ctaid_z_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%cluster_nctaid.x; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_cluster_nctaid_x();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_nctaid_x_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_nctaid_x()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%cluster_nctaid.x;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_cluster_nctaid_x_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%cluster_nctaid.y; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_cluster_nctaid_y();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_nctaid_y_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_nctaid_y()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%cluster_nctaid.y;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_cluster_nctaid_y_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%cluster_nctaid.z; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_cluster_nctaid_z();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_nctaid_z_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_nctaid_z()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%cluster_nctaid.z;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_cluster_nctaid_z_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%cluster_ctarank; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_cluster_ctarank();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_ctarank_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_ctarank()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%cluster_ctarank;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_cluster_ctarank_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%cluster_nctarank; // PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_cluster_nctarank();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_nctarank_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_nctarank()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%cluster_nctarank;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_cluster_nctarank_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%lanemask_eq; // PTX ISA 20, SM_35
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_lanemask_eq();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_eq_is_not_supported_before_SM_35__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_eq()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%lanemask_eq;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_lanemask_eq_is_not_supported_before_SM_35__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%lanemask_le; // PTX ISA 20, SM_35
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_lanemask_le();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_le_is_not_supported_before_SM_35__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_le()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%lanemask_le;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_lanemask_le_is_not_supported_before_SM_35__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%lanemask_lt; // PTX ISA 20, SM_35
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_lanemask_lt();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_lt_is_not_supported_before_SM_35__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_lt()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%lanemask_lt;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_lanemask_lt_is_not_supported_before_SM_35__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%lanemask_ge; // PTX ISA 20, SM_35
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_lanemask_ge();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_ge_is_not_supported_before_SM_35__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_ge()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%lanemask_ge;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_lanemask_ge_is_not_supported_before_SM_35__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%lanemask_gt; // PTX ISA 20, SM_35
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_lanemask_gt();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_gt_is_not_supported_before_SM_35__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_gt()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%lanemask_gt;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_lanemask_gt_is_not_supported_before_SM_35__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%clock; // PTX ISA 10
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_clock();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 100
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clock()
|
||||
{
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm volatile("mov.u32 %0, %%clock;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 100
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%clock_hi; // PTX ISA 50, SM_35
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_clock_hi();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 500
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clock_hi_is_not_supported_before_SM_35__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clock_hi()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm volatile("mov.u32 %0, %%clock_hi;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_clock_hi_is_not_supported_before_SM_35__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 500
|
||||
|
||||
/*
|
||||
// mov.u64 sreg_value, %%clock64; // PTX ISA 20, SM_35
|
||||
template <typename = void>
|
||||
__device__ static inline uint64_t get_sreg_clock64();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clock64_is_not_supported_before_SM_35__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint64_t get_sreg_clock64()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
|
||||
::cuda::std::uint64_t __sreg_value;
|
||||
asm volatile("mov.u64 %0, %%clock64;" : "=l"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_clock64_is_not_supported_before_SM_35__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// mov.u64 sreg_value, %%globaltimer; // PTX ISA 31, SM_35
|
||||
template <typename = void>
|
||||
__device__ static inline uint64_t get_sreg_globaltimer();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 310
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_globaltimer_is_not_supported_before_SM_35__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint64_t get_sreg_globaltimer()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
|
||||
::cuda::std::uint64_t __sreg_value;
|
||||
asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_globaltimer_is_not_supported_before_SM_35__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 310
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%globaltimer_lo; // PTX ISA 31, SM_35
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_globaltimer_lo();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 310
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_globaltimer_lo_is_not_supported_before_SM_35__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_globaltimer_lo()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm volatile("mov.u32 %0, %%globaltimer_lo;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_globaltimer_lo_is_not_supported_before_SM_35__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 310
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%globaltimer_hi; // PTX ISA 31, SM_35
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_globaltimer_hi();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 310
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_globaltimer_hi_is_not_supported_before_SM_35__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_globaltimer_hi()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm volatile("mov.u32 %0, %%globaltimer_hi;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_globaltimer_hi_is_not_supported_before_SM_35__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 310
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%total_smem_size; // PTX ISA 41, SM_35
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_total_smem_size();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 410
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_total_smem_size_is_not_supported_before_SM_35__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_total_smem_size()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%total_smem_size;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_total_smem_size_is_not_supported_before_SM_35__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 410
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%aggr_smem_size; // PTX ISA 81, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_aggr_smem_size();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_aggr_smem_size_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_aggr_smem_size()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%aggr_smem_size;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_aggr_smem_size_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// mov.u32 sreg_value, %%dynamic_smem_size; // PTX ISA 41, SM_35
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t get_sreg_dynamic_smem_size();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 410
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_dynamic_smem_size_is_not_supported_before_SM_35__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_dynamic_smem_size()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
|
||||
::cuda::std::uint32_t __sreg_value;
|
||||
asm("mov.u32 %0, %%dynamic_smem_size;" : "=r"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_dynamic_smem_size_is_not_supported_before_SM_35__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 410
|
||||
|
||||
/*
|
||||
// mov.u64 sreg_value, %%current_graph_exec; // PTX ISA 80, SM_50
|
||||
template <typename = void>
|
||||
__device__ static inline uint64_t get_sreg_current_graph_exec();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_current_graph_exec_is_not_supported_before_SM_50__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint64_t get_sreg_current_graph_exec()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
|
||||
::cuda::std::uint64_t __sreg_value;
|
||||
asm("mov.u64 %0, %%current_graph_exec;" : "=l"(__sreg_value) : :);
|
||||
return __sreg_value;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_get_sreg_current_graph_exec_is_not_supported_before_SM_50__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_GET_SREG_H_
|
||||
@@ -0,0 +1,32 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_GETCTARANK_H_
|
||||
#define _CUDA_PTX_GENERATED_GETCTARANK_H_
|
||||
|
||||
/*
|
||||
// getctarank.space.u32 dest, addr; // PTX ISA 78, SM_90
|
||||
// .space = { .shared::cluster }
|
||||
template <typename = void>
|
||||
__device__ static inline uint32_t getctarank(
|
||||
cuda::ptx::space_cluster_t,
|
||||
const void* addr);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_getctarank_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t getctarank(::cuda::ptx::space_cluster_t, const void* __addr)
|
||||
{
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("getctarank.shared::cluster.u32 %0, %1;" : "=r"(__dest) : "r"(__as_ptr_smem(__addr)) :);
|
||||
return __dest;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_getctarank_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_GETCTARANK_H_
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,399 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_H_
|
||||
#define _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_H_
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.shared.b64 state, [addr]; // 1. PTX ISA 70, SM_80
|
||||
template <typename = void>
|
||||
__device__ static inline uint64_t mbarrier_arrive(
|
||||
uint64_t* addr);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 700
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_80__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint64_t mbarrier_arrive(::cuda::std::uint64_t* __addr)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 800
|
||||
::cuda::std::uint64_t __state;
|
||||
asm("mbarrier.arrive.shared.b64 %0, [%1]; // 1. "
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_smem(__addr))
|
||||
: "memory");
|
||||
return __state;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_80__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 700
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.shared::cta.b64 state, [addr], count; // 2. PTX ISA 78, SM_90
|
||||
template <typename = void>
|
||||
__device__ static inline uint64_t mbarrier_arrive(
|
||||
uint64_t* addr,
|
||||
const uint32_t& count);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint64_t
|
||||
mbarrier_arrive(::cuda::std::uint64_t* __addr, const ::cuda::std::uint32_t& __count)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint64_t __state;
|
||||
asm("mbarrier.arrive.shared::cta.b64 %0, [%1], %2; // 2. "
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__count)
|
||||
: "memory");
|
||||
return __state;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.sem.scope.space.b64 state, [addr]; // 3a. PTX ISA 80, SM_90
|
||||
// .sem = { .release }
|
||||
// .scope = { .cta, .cluster }
|
||||
// .space = { .shared::cta }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline uint64_t mbarrier_arrive(
|
||||
cuda::ptx::sem_release_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
cuda::ptx::space_shared_t,
|
||||
uint64_t* addr);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint64_t mbarrier_arrive(
|
||||
::cuda::ptx::sem_release_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::std::uint64_t* __addr)
|
||||
{
|
||||
// __sem == sem_release (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint64_t __state;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("mbarrier.arrive.release.cta.shared::cta.b64 %0, [%1]; // 3a. "
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_smem(__addr))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("mbarrier.arrive.release.cluster.shared::cta.b64 %0, [%1]; // 3a. "
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_smem(__addr))
|
||||
: "memory");
|
||||
}
|
||||
return __state;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.sem.scope.space.b64 state, [addr], count; // 3b. PTX ISA 80, SM_90
|
||||
// .sem = { .release }
|
||||
// .scope = { .cta, .cluster }
|
||||
// .space = { .shared::cta }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline uint64_t mbarrier_arrive(
|
||||
cuda::ptx::sem_release_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
cuda::ptx::space_shared_t,
|
||||
uint64_t* addr,
|
||||
const uint32_t& count);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint64_t mbarrier_arrive(
|
||||
::cuda::ptx::sem_release_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __count)
|
||||
{
|
||||
// __sem == sem_release (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint64_t __state;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("mbarrier.arrive.release.cta.shared::cta.b64 %0, [%1], %2; // 3b. "
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__count)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("mbarrier.arrive.release.cluster.shared::cta.b64 %0, [%1], %2; // 3b. "
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__count)
|
||||
: "memory");
|
||||
}
|
||||
return __state;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.sem.scope.space.b64 _, [addr]; // 4a. PTX ISA 80, SM_90
|
||||
// .sem = { .release }
|
||||
// .scope = { .cluster }
|
||||
// .space = { .shared::cluster }
|
||||
template <typename = void>
|
||||
__device__ static inline void mbarrier_arrive(
|
||||
cuda::ptx::sem_release_t,
|
||||
cuda::ptx::scope_cluster_t,
|
||||
cuda::ptx::space_cluster_t,
|
||||
uint64_t* addr);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void mbarrier_arrive(
|
||||
::cuda::ptx::sem_release_t, ::cuda::ptx::scope_cluster_t, ::cuda::ptx::space_cluster_t, ::cuda::std::uint64_t* __addr)
|
||||
{
|
||||
// __sem == sem_release (due to parameter type constraint)
|
||||
// __scope == scope_cluster (due to parameter type constraint)
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("mbarrier.arrive.release.cluster.shared::cluster.b64 _, [%0]; // 4a. "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__addr))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.sem.scope.space.b64 _, [addr], count; // 4b. PTX ISA 80, SM_90
|
||||
// .sem = { .release }
|
||||
// .scope = { .cluster }
|
||||
// .space = { .shared::cluster }
|
||||
template <typename = void>
|
||||
__device__ static inline void mbarrier_arrive(
|
||||
cuda::ptx::sem_release_t,
|
||||
cuda::ptx::scope_cluster_t,
|
||||
cuda::ptx::space_cluster_t,
|
||||
uint64_t* addr,
|
||||
const uint32_t& count);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void mbarrier_arrive(
|
||||
::cuda::ptx::sem_release_t,
|
||||
::cuda::ptx::scope_cluster_t,
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __count)
|
||||
{
|
||||
// __sem == sem_release (due to parameter type constraint)
|
||||
// __scope == scope_cluster (due to parameter type constraint)
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("mbarrier.arrive.release.cluster.shared::cluster.b64 _, [%0], %1; // 4b. "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__addr)), "r"(__count)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.sem.scope.space.b64 state, [addr], count; // PTX ISA 86, SM_90
|
||||
// .sem = { .relaxed }
|
||||
// .scope = { .cta, .cluster }
|
||||
// .space = { .shared::cta }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline uint64_t mbarrier_arrive(
|
||||
cuda::ptx::sem_relaxed_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
cuda::ptx::space_shared_t,
|
||||
uint64_t* addr,
|
||||
const uint32_t& count);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint64_t mbarrier_arrive(
|
||||
::cuda::ptx::sem_relaxed_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __count)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint64_t __state;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("mbarrier.arrive.relaxed.cta.shared::cta.b64 %0, [%1], %2;"
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__count)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("mbarrier.arrive.relaxed.cluster.shared::cta.b64 %0, [%1], %2;"
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__count)
|
||||
: "memory");
|
||||
}
|
||||
return __state;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.sem.scope.space.b64 state, [addr]; // PTX ISA 86, SM_90
|
||||
// .sem = { .relaxed }
|
||||
// .scope = { .cta, .cluster }
|
||||
// .space = { .shared::cta }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline uint64_t mbarrier_arrive(
|
||||
cuda::ptx::sem_relaxed_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
cuda::ptx::space_shared_t,
|
||||
uint64_t* addr);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint64_t mbarrier_arrive(
|
||||
::cuda::ptx::sem_relaxed_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::std::uint64_t* __addr)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint64_t __state;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("mbarrier.arrive.relaxed.cta.shared::cta.b64 %0, [%1];"
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_smem(__addr))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("mbarrier.arrive.relaxed.cluster.shared::cta.b64 %0, [%1];"
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_smem(__addr))
|
||||
: "memory");
|
||||
}
|
||||
return __state;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.sem.scope.space.b64 _, [addr], count; // PTX ISA 86, SM_90
|
||||
// .sem = { .relaxed }
|
||||
// .scope = { .cluster }
|
||||
// .space = { .shared::cluster }
|
||||
template <typename = void>
|
||||
__device__ static inline void mbarrier_arrive(
|
||||
cuda::ptx::sem_relaxed_t,
|
||||
cuda::ptx::scope_cluster_t,
|
||||
cuda::ptx::space_cluster_t,
|
||||
uint64_t* addr,
|
||||
const uint32_t& count);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void mbarrier_arrive(
|
||||
::cuda::ptx::sem_relaxed_t,
|
||||
::cuda::ptx::scope_cluster_t,
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __count)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
// __scope == scope_cluster (due to parameter type constraint)
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("mbarrier.arrive.relaxed.cluster.shared::cluster.b64 _, [%0], %1;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__count)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.sem.scope.space.b64 _, [addr]; // PTX ISA 86, SM_90
|
||||
// .sem = { .relaxed }
|
||||
// .scope = { .cluster }
|
||||
// .space = { .shared::cluster }
|
||||
template <typename = void>
|
||||
__device__ static inline void mbarrier_arrive(
|
||||
cuda::ptx::sem_relaxed_t,
|
||||
cuda::ptx::scope_cluster_t,
|
||||
cuda::ptx::space_cluster_t,
|
||||
uint64_t* addr);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void mbarrier_arrive(
|
||||
::cuda::ptx::sem_relaxed_t, ::cuda::ptx::scope_cluster_t, ::cuda::ptx::space_cluster_t, ::cuda::std::uint64_t* __addr)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
// __scope == scope_cluster (due to parameter type constraint)
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("mbarrier.arrive.relaxed.cluster.shared::cluster.b64 _, [%0];" : : "r"(__as_ptr_smem(__addr)) : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_H_
|
||||
@@ -0,0 +1,184 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_EXPECT_TX_H_
|
||||
#define _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_EXPECT_TX_H_
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.expect_tx.sem.scope.space.b64 state, [addr], tx_count; // 8. PTX ISA 80, SM_90
|
||||
// .sem = { .release }
|
||||
// .scope = { .cta, .cluster }
|
||||
// .space = { .shared::cta }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline uint64_t mbarrier_arrive_expect_tx(
|
||||
cuda::ptx::sem_release_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
cuda::ptx::space_shared_t,
|
||||
uint64_t* addr,
|
||||
const uint32_t& tx_count);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint64_t mbarrier_arrive_expect_tx(
|
||||
::cuda::ptx::sem_release_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __tx_count)
|
||||
{
|
||||
// __sem == sem_release (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint64_t __state;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("mbarrier.arrive.expect_tx.release.cta.shared::cta.b64 %0, [%1], %2; // 8. "
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__tx_count)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("mbarrier.arrive.expect_tx.release.cluster.shared::cta.b64 %0, [%1], %2; // 8. "
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__tx_count)
|
||||
: "memory");
|
||||
}
|
||||
return __state;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.expect_tx.sem.scope.space.b64 _, [addr], tx_count; // 9. PTX ISA 80, SM_90
|
||||
// .sem = { .release }
|
||||
// .scope = { .cluster }
|
||||
// .space = { .shared::cluster }
|
||||
template <typename = void>
|
||||
__device__ static inline void mbarrier_arrive_expect_tx(
|
||||
cuda::ptx::sem_release_t,
|
||||
cuda::ptx::scope_cluster_t,
|
||||
cuda::ptx::space_cluster_t,
|
||||
uint64_t* addr,
|
||||
const uint32_t& tx_count);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void mbarrier_arrive_expect_tx(
|
||||
::cuda::ptx::sem_release_t,
|
||||
::cuda::ptx::scope_cluster_t,
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __tx_count)
|
||||
{
|
||||
// __sem == sem_release (due to parameter type constraint)
|
||||
// __scope == scope_cluster (due to parameter type constraint)
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("mbarrier.arrive.expect_tx.release.cluster.shared::cluster.b64 _, [%0], %1; // 9. "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__addr)), "r"(__tx_count)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.expect_tx.sem.scope.space.b64 state, [addr], txCount; // PTX ISA 86, SM_90
|
||||
// .sem = { .relaxed }
|
||||
// .scope = { .cta, .cluster }
|
||||
// .space = { .shared::cta }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline uint64_t mbarrier_arrive_expect_tx(
|
||||
cuda::ptx::sem_relaxed_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
cuda::ptx::space_shared_t,
|
||||
uint64_t* addr,
|
||||
const uint32_t& txCount);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint64_t mbarrier_arrive_expect_tx(
|
||||
::cuda::ptx::sem_relaxed_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __txCount)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint64_t __state;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("mbarrier.arrive.expect_tx.relaxed.cta.shared::cta.b64 %0, [%1], %2;"
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_dsmem(__addr)), "r"(__txCount)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("mbarrier.arrive.expect_tx.relaxed.cluster.shared::cta.b64 %0, [%1], %2;"
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_dsmem(__addr)), "r"(__txCount)
|
||||
: "memory");
|
||||
}
|
||||
return __state;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.expect_tx.sem.scope.space.b64 _, [addr], txCount; // PTX ISA 86, SM_90
|
||||
// .sem = { .relaxed }
|
||||
// .scope = { .cluster }
|
||||
// .space = { .shared::cluster }
|
||||
template <typename = void>
|
||||
__device__ static inline void mbarrier_arrive_expect_tx(
|
||||
cuda::ptx::sem_relaxed_t,
|
||||
cuda::ptx::scope_cluster_t,
|
||||
cuda::ptx::space_cluster_t,
|
||||
uint64_t* addr,
|
||||
const uint32_t& txCount);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void mbarrier_arrive_expect_tx(
|
||||
::cuda::ptx::sem_relaxed_t,
|
||||
::cuda::ptx::scope_cluster_t,
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __txCount)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
// __scope == scope_cluster (due to parameter type constraint)
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("mbarrier.arrive.expect_tx.relaxed.cluster.shared::cluster.b64 _, [%0], %1;"
|
||||
:
|
||||
: "r"(__as_ptr_dsmem(__addr)), "r"(__txCount)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_EXPECT_TX_H_
|
||||
@@ -0,0 +1,34 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_NO_COMPLETE_H_
|
||||
#define _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_NO_COMPLETE_H_
|
||||
|
||||
/*
|
||||
// mbarrier.arrive.noComplete.shared.b64 state, [addr], count; // 5. PTX ISA 70, SM_80
|
||||
template <typename = void>
|
||||
__device__ static inline uint64_t mbarrier_arrive_no_complete(
|
||||
uint64_t* addr,
|
||||
const uint32_t& count);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 700
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_arrive_no_complete_is_not_supported_before_SM_80__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint64_t
|
||||
mbarrier_arrive_no_complete(::cuda::std::uint64_t* __addr, const ::cuda::std::uint32_t& __count)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 800
|
||||
::cuda::std::uint64_t __state;
|
||||
asm("mbarrier.arrive.noComplete.shared.b64 %0, [%1], %2; // 5. "
|
||||
: "=l"(__state)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__count)
|
||||
: "memory");
|
||||
return __state;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_arrive_no_complete_is_not_supported_before_SM_80__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 700
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_MBARRIER_ARRIVE_NO_COMPLETE_H_
|
||||
@@ -0,0 +1,102 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_MBARRIER_EXPECT_TX_H_
|
||||
#define _CUDA_PTX_GENERATED_MBARRIER_EXPECT_TX_H_
|
||||
|
||||
/*
|
||||
// mbarrier.expect_tx.sem.scope.space.b64 [addr], txCount; // 1. PTX ISA 80, SM_90
|
||||
// .sem = { .relaxed }
|
||||
// .scope = { .cta, .cluster }
|
||||
// .space = { .shared::cta }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline void mbarrier_expect_tx(
|
||||
cuda::ptx::sem_relaxed_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
cuda::ptx::space_shared_t,
|
||||
uint64_t* addr,
|
||||
uint32_t txCount);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_expect_tx_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline void mbarrier_expect_tx(
|
||||
::cuda::ptx::sem_relaxed_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::ptx::space_shared_t,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
::cuda::std::uint32_t __txCount)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("mbarrier.expect_tx.relaxed.cta.shared::cta.b64 [%0], %1; // 1."
|
||||
:
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__txCount)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("mbarrier.expect_tx.relaxed.cluster.shared::cta.b64 [%0], %1; // 1."
|
||||
:
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__txCount)
|
||||
: "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_expect_tx_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// mbarrier.expect_tx.sem.scope.space.b64 [addr], txCount; // 2. PTX ISA 80, SM_90
|
||||
// .sem = { .relaxed }
|
||||
// .scope = { .cta, .cluster }
|
||||
// .space = { .shared::cluster }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline void mbarrier_expect_tx(
|
||||
cuda::ptx::sem_relaxed_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
cuda::ptx::space_cluster_t,
|
||||
uint64_t* addr,
|
||||
uint32_t txCount);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_expect_tx_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline void mbarrier_expect_tx(
|
||||
::cuda::ptx::sem_relaxed_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::ptx::space_cluster_t,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
::cuda::std::uint32_t __txCount)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
// __space == space_cluster (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("mbarrier.expect_tx.relaxed.cta.shared::cluster.b64 [%0], %1; // 2."
|
||||
:
|
||||
: "r"(__as_ptr_dsmem(__addr)), "r"(__txCount)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("mbarrier.expect_tx.relaxed.cluster.shared::cluster.b64 [%0], %1; // 2."
|
||||
:
|
||||
: "r"(__as_ptr_dsmem(__addr)), "r"(__txCount)
|
||||
: "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_expect_tx_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_MBARRIER_EXPECT_TX_H_
|
||||
@@ -0,0 +1,27 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_MBARRIER_INIT_H_
|
||||
#define _CUDA_PTX_GENERATED_MBARRIER_INIT_H_
|
||||
|
||||
/*
|
||||
// mbarrier.init.shared.b64 [addr], count; // PTX ISA 70, SM_80
|
||||
template <typename = void>
|
||||
__device__ static inline void mbarrier_init(
|
||||
uint64_t* addr,
|
||||
const uint32_t& count);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 700
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_init_is_not_supported_before_SM_80__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void mbarrier_init(::cuda::std::uint64_t* __addr, const ::cuda::std::uint32_t& __count)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 800
|
||||
asm("mbarrier.init.shared.b64 [%0], %1;" : : "r"(__as_ptr_smem(__addr)), "r"(__count) : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_init_is_not_supported_before_SM_80__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 700
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_MBARRIER_INIT_H_
|
||||
@@ -0,0 +1,26 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_MBARRIER_INVAL_H_
|
||||
#define _CUDA_PTX_GENERATED_MBARRIER_INVAL_H_
|
||||
|
||||
/*
|
||||
// mbarrier.inval.shared.b64 [addr]; // PTX ISA 70, SM_80
|
||||
template <typename = void>
|
||||
__device__ static inline void mbarrier_inval(
|
||||
uint64_t* addr);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 700
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_inval_is_not_supported_before_SM_80__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void mbarrier_inval(::cuda::std::uint64_t* __addr)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 800
|
||||
asm("mbarrier.inval.shared.b64 [%0];" : : "r"(__as_ptr_smem(__addr)) : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_inval_is_not_supported_before_SM_80__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 700
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_MBARRIER_INVAL_H_
|
||||
@@ -0,0 +1,143 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_MBARRIER_TEST_WAIT_H_
|
||||
#define _CUDA_PTX_GENERATED_MBARRIER_TEST_WAIT_H_
|
||||
|
||||
/*
|
||||
// mbarrier.test_wait.shared.b64 waitComplete, [addr], state; // 1. PTX
|
||||
ISA 70, SM_80 template <typename = void>
|
||||
__device__ static inline bool mbarrier_test_wait(
|
||||
uint64_t* addr,
|
||||
const uint64_t& state);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 700
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_80__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline bool mbarrier_test_wait(::cuda::std::uint64_t* __addr, const ::cuda::std::uint64_t& __state)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 800
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.test_wait.shared.b64 P_OUT, [%1], %2; // 1. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state)
|
||||
: "memory");
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_80__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 700
|
||||
|
||||
/*
|
||||
// mbarrier.test_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state; // 2. PTX
|
||||
ISA 80, SM_90
|
||||
// .sem = { .acquire }
|
||||
// .scope = { .cta, .cluster }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline bool mbarrier_test_wait(
|
||||
cuda::ptx::sem_acquire_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
uint64_t* addr,
|
||||
const uint64_t& state);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline bool mbarrier_test_wait(
|
||||
::cuda::ptx::sem_acquire_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint64_t& __state)
|
||||
{
|
||||
// __sem == sem_acquire (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.test_wait.acquire.cta.shared::cta.b64 P_OUT, [%1], %2; // 2. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.test_wait.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2; // 2. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state)
|
||||
: "memory");
|
||||
}
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// mbarrier.test_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state; // PTX ISA 86, SM_90
|
||||
// .sem = { .relaxed }
|
||||
// .scope = { .cta, .cluster }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline bool mbarrier_test_wait(
|
||||
cuda::ptx::sem_relaxed_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
uint64_t* addr,
|
||||
const uint64_t& state);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline bool mbarrier_test_wait(
|
||||
::cuda::ptx::sem_relaxed_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint64_t& __state)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.test_wait.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.test_wait.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state)
|
||||
: "memory");
|
||||
}
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_MBARRIER_TEST_WAIT_H_
|
||||
@@ -0,0 +1,144 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_MBARRIER_TEST_WAIT_PARITY_H_
|
||||
#define _CUDA_PTX_GENERATED_MBARRIER_TEST_WAIT_PARITY_H_
|
||||
|
||||
/*
|
||||
// mbarrier.test_wait.parity.shared.b64 waitComplete, [addr], phaseParity; // 3. PTX
|
||||
ISA 71, SM_80 template <typename = void>
|
||||
__device__ static inline bool mbarrier_test_wait_parity(
|
||||
uint64_t* addr,
|
||||
const uint32_t& phaseParity);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 710
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_80__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline bool
|
||||
mbarrier_test_wait_parity(::cuda::std::uint64_t* __addr, const ::cuda::std::uint32_t& __phaseParity)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 800
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.test_wait.parity.shared.b64 P_OUT, [%1], %2; // 3. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
|
||||
: "memory");
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_80__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 710
|
||||
|
||||
/*
|
||||
// mbarrier.test_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity; // 4. PTX
|
||||
ISA 80, SM_90
|
||||
// .sem = { .acquire }
|
||||
// .scope = { .cta, .cluster }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline bool mbarrier_test_wait_parity(
|
||||
cuda::ptx::sem_acquire_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
uint64_t* addr,
|
||||
const uint32_t& phaseParity);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline bool mbarrier_test_wait_parity(
|
||||
::cuda::ptx::sem_acquire_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __phaseParity)
|
||||
{
|
||||
// __sem == sem_acquire (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.test_wait.parity.acquire.cta.shared::cta.b64 P_OUT, [%1], %2; // 4. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.test_wait.parity.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2; // 4. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
|
||||
: "memory");
|
||||
}
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// mbarrier.test_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity; // PTX ISA 86, SM_90
|
||||
// .sem = { .relaxed }
|
||||
// .scope = { .cta, .cluster }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline bool mbarrier_test_wait_parity(
|
||||
cuda::ptx::sem_relaxed_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
uint64_t* addr,
|
||||
const uint32_t& phaseParity);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline bool mbarrier_test_wait_parity(
|
||||
::cuda::ptx::sem_relaxed_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __phaseParity)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.test_wait.parity.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.test_wait.parity.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
|
||||
: "memory");
|
||||
}
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_MBARRIER_TEST_WAIT_PARITY_H_
|
||||
@@ -0,0 +1,286 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_MBARRIER_TRY_WAIT_H_
|
||||
#define _CUDA_PTX_GENERATED_MBARRIER_TRY_WAIT_H_
|
||||
|
||||
/*
|
||||
// mbarrier.try_wait.shared::cta.b64 waitComplete, [addr], state; // 5a.
|
||||
PTX ISA 78, SM_90 template <typename = void>
|
||||
__device__ static inline bool mbarrier_try_wait(
|
||||
uint64_t* addr,
|
||||
const uint64_t& state);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline bool mbarrier_try_wait(::cuda::std::uint64_t* __addr, const ::cuda::std::uint64_t& __state)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.shared::cta.b64 P_OUT, [%1], %2; // 5a. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state)
|
||||
: "memory");
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mbarrier.try_wait.shared::cta.b64 waitComplete, [addr], state, suspendTimeHint; // 5b. PTX
|
||||
ISA 78, SM_90 template <typename = void>
|
||||
__device__ static inline bool mbarrier_try_wait(
|
||||
uint64_t* addr,
|
||||
const uint64_t& state,
|
||||
const uint32_t& suspendTimeHint);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline bool mbarrier_try_wait(
|
||||
::cuda::std::uint64_t* __addr, const ::cuda::std::uint64_t& __state, const ::cuda::std::uint32_t& __suspendTimeHint)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.shared::cta.b64 P_OUT, [%1], %2, %3; // 5b. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state), "r"(__suspendTimeHint)
|
||||
: "memory");
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mbarrier.try_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state; // 6a. PTX
|
||||
ISA 80, SM_90
|
||||
// .sem = { .acquire }
|
||||
// .scope = { .cta, .cluster }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline bool mbarrier_try_wait(
|
||||
cuda::ptx::sem_acquire_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
uint64_t* addr,
|
||||
const uint64_t& state);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline bool mbarrier_try_wait(
|
||||
::cuda::ptx::sem_acquire_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint64_t& __state)
|
||||
{
|
||||
// __sem == sem_acquire (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.acquire.cta.shared::cta.b64 P_OUT, [%1], %2; // 6a. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2; // 6a. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state)
|
||||
: "memory");
|
||||
}
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// mbarrier.try_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state , suspendTimeHint; // 6b. PTX
|
||||
ISA 80, SM_90
|
||||
// .sem = { .acquire }
|
||||
// .scope = { .cta, .cluster }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline bool mbarrier_try_wait(
|
||||
cuda::ptx::sem_acquire_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
uint64_t* addr,
|
||||
const uint64_t& state,
|
||||
const uint32_t& suspendTimeHint);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline bool mbarrier_try_wait(
|
||||
::cuda::ptx::sem_acquire_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint64_t& __state,
|
||||
const ::cuda::std::uint32_t& __suspendTimeHint)
|
||||
{
|
||||
// __sem == sem_acquire (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.acquire.cta.shared::cta.b64 P_OUT, [%1], %2 , %3; // 6b. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state), "r"(__suspendTimeHint)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2 , %3; // 6b. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state), "r"(__suspendTimeHint)
|
||||
: "memory");
|
||||
}
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// mbarrier.try_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state, suspendTimeHint; // PTX ISA 86, SM_90
|
||||
// .sem = { .relaxed }
|
||||
// .scope = { .cta, .cluster }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline bool mbarrier_try_wait(
|
||||
cuda::ptx::sem_relaxed_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
uint64_t* addr,
|
||||
const uint64_t& state,
|
||||
const uint32_t& suspendTimeHint);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline bool mbarrier_try_wait(
|
||||
::cuda::ptx::sem_relaxed_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint64_t& __state,
|
||||
const ::cuda::std::uint32_t& __suspendTimeHint)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2, %3;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state), "r"(__suspendTimeHint)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2, %3;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state), "r"(__suspendTimeHint)
|
||||
: "memory");
|
||||
}
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// mbarrier.try_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state; // PTX ISA 86, SM_90
|
||||
// .sem = { .relaxed }
|
||||
// .scope = { .cta, .cluster }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline bool mbarrier_try_wait(
|
||||
cuda::ptx::sem_relaxed_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
uint64_t* addr,
|
||||
const uint64_t& state);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline bool mbarrier_try_wait(
|
||||
::cuda::ptx::sem_relaxed_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint64_t& __state)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__state)
|
||||
: "memory");
|
||||
}
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_MBARRIER_TRY_WAIT_H_
|
||||
@@ -0,0 +1,290 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_MBARRIER_TRY_WAIT_PARITY_H_
|
||||
#define _CUDA_PTX_GENERATED_MBARRIER_TRY_WAIT_PARITY_H_
|
||||
|
||||
/*
|
||||
// mbarrier.try_wait.parity.shared::cta.b64 waitComplete, [addr], phaseParity; // 7a.
|
||||
PTX ISA 78, SM_90 template <typename = void>
|
||||
__device__ static inline bool mbarrier_try_wait_parity(
|
||||
uint64_t* addr,
|
||||
const uint32_t& phaseParity);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline bool
|
||||
mbarrier_try_wait_parity(::cuda::std::uint64_t* __addr, const ::cuda::std::uint32_t& __phaseParity)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.parity.shared::cta.b64 P_OUT, [%1], %2; // 7a. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
|
||||
: "memory");
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mbarrier.try_wait.parity.shared::cta.b64 waitComplete, [addr], phaseParity, suspendTimeHint; // 7b.
|
||||
PTX ISA 78, SM_90 template <typename = void>
|
||||
__device__ static inline bool mbarrier_try_wait_parity(
|
||||
uint64_t* addr,
|
||||
const uint32_t& phaseParity,
|
||||
const uint32_t& suspendTimeHint);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 780
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline bool mbarrier_try_wait_parity(
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __phaseParity,
|
||||
const ::cuda::std::uint32_t& __suspendTimeHint)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.parity.shared::cta.b64 P_OUT, [%1], %2, %3; // 7b. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity), "r"(__suspendTimeHint)
|
||||
: "memory");
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 780
|
||||
|
||||
/*
|
||||
// mbarrier.try_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity; // 8a. PTX
|
||||
ISA 80, SM_90
|
||||
// .sem = { .acquire }
|
||||
// .scope = { .cta, .cluster }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline bool mbarrier_try_wait_parity(
|
||||
cuda::ptx::sem_acquire_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
uint64_t* addr,
|
||||
const uint32_t& phaseParity);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline bool mbarrier_try_wait_parity(
|
||||
::cuda::ptx::sem_acquire_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __phaseParity)
|
||||
{
|
||||
// __sem == sem_acquire (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.parity.acquire.cta.shared::cta.b64 P_OUT, [%1], %2; // 8a. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2; // 8a. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
|
||||
: "memory");
|
||||
}
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// mbarrier.try_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity, suspendTimeHint; // 8b. PTX
|
||||
ISA 80, SM_90
|
||||
// .sem = { .acquire }
|
||||
// .scope = { .cta, .cluster }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline bool mbarrier_try_wait_parity(
|
||||
cuda::ptx::sem_acquire_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
uint64_t* addr,
|
||||
const uint32_t& phaseParity,
|
||||
const uint32_t& suspendTimeHint);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline bool mbarrier_try_wait_parity(
|
||||
::cuda::ptx::sem_acquire_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __phaseParity,
|
||||
const ::cuda::std::uint32_t& __suspendTimeHint)
|
||||
{
|
||||
// __sem == sem_acquire (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.parity.acquire.cta.shared::cta.b64 P_OUT, [%1], %2, %3; // 8b. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity), "r"(__suspendTimeHint)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2, %3; // 8b. \n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity), "r"(__suspendTimeHint)
|
||||
: "memory");
|
||||
}
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// mbarrier.try_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity, suspendTimeHint; // PTX ISA 86,
|
||||
SM_90
|
||||
// .sem = { .relaxed }
|
||||
// .scope = { .cta, .cluster }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline bool mbarrier_try_wait_parity(
|
||||
cuda::ptx::sem_relaxed_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
uint64_t* addr,
|
||||
const uint32_t& phaseParity,
|
||||
const uint32_t& suspendTimeHint);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline bool mbarrier_try_wait_parity(
|
||||
::cuda::ptx::sem_relaxed_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __phaseParity,
|
||||
const ::cuda::std::uint32_t& __suspendTimeHint)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.parity.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2, %3;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity), "r"(__suspendTimeHint)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.parity.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2, %3;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity), "r"(__suspendTimeHint)
|
||||
: "memory");
|
||||
}
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// mbarrier.try_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity; // PTX ISA 86, SM_90
|
||||
// .sem = { .relaxed }
|
||||
// .scope = { .cta, .cluster }
|
||||
template <cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline bool mbarrier_try_wait_parity(
|
||||
cuda::ptx::sem_relaxed_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
uint64_t* addr,
|
||||
const uint32_t& phaseParity);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
|
||||
template <::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline bool mbarrier_try_wait_parity(
|
||||
::cuda::ptx::sem_relaxed_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
::cuda::std::uint64_t* __addr,
|
||||
const ::cuda::std::uint32_t& __phaseParity)
|
||||
{
|
||||
// __sem == sem_relaxed (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
::cuda::std::uint32_t __waitComplete;
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.parity.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm("{\n\t .reg .pred P_OUT; \n\t"
|
||||
"mbarrier.try_wait.parity.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2;\n\t"
|
||||
"selp.b32 %0, 1, 0, P_OUT; \n"
|
||||
"}"
|
||||
: "=r"(__waitComplete)
|
||||
: "r"(__as_ptr_smem(__addr)), "r"(__phaseParity)
|
||||
: "memory");
|
||||
}
|
||||
return static_cast<bool>(__waitComplete);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
|
||||
return false;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_MBARRIER_TRY_WAIT_PARITY_H_
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,236 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_MULTIMEM_ST_H_
|
||||
#define _CUDA_PTX_GENERATED_MULTIMEM_ST_H_
|
||||
|
||||
/*
|
||||
// multimem.st.sem.global.b32 [addr], val; // PTX ISA 81, SM_90
|
||||
// .sem = { .weak }
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void multimem_st(
|
||||
cuda::ptx::sem_weak_t,
|
||||
B32* addr,
|
||||
B32 val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void multimem_st(::cuda::ptx::sem_weak_t, _B32* __addr, _B32 __val)
|
||||
{
|
||||
// __sem == sem_weak (due to parameter type constraint)
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("multimem.st.weak.global.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// multimem.st.sem.scope.global.b32 [addr], val; // PTX ISA 81, SM_90
|
||||
// .sem = { .relaxed, .release }
|
||||
// .scope = { .cta, .cluster, .gpu, .sys }
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true, cuda::ptx::dot_sem Sem, cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline void multimem_st(
|
||||
cuda::ptx::sem_t<Sem> sem,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
B32* addr,
|
||||
B32 val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
|
||||
template <typename _B32,
|
||||
::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true,
|
||||
::cuda::ptx::dot_sem _Sem,
|
||||
::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline void
|
||||
multimem_st(::cuda::ptx::sem_t<_Sem> __sem, ::cuda::ptx::scope_t<_Scope> __scope, _B32* __addr, _B32 __val)
|
||||
{
|
||||
static_assert(__sem == sem_relaxed || __sem == sem_release);
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster || __scope == scope_gpu || __scope == scope_sys);
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__sem == sem_relaxed && __scope == scope_cta)
|
||||
{
|
||||
asm("multimem.st.relaxed.cta.global.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__sem == sem_relaxed && __scope == scope_cluster)
|
||||
{
|
||||
asm("multimem.st.relaxed.cluster.global.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__sem == sem_relaxed && __scope == scope_gpu)
|
||||
{
|
||||
asm("multimem.st.relaxed.gpu.global.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__sem == sem_relaxed && __scope == scope_sys)
|
||||
{
|
||||
asm("multimem.st.relaxed.sys.global.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__sem == sem_release && __scope == scope_cta)
|
||||
{
|
||||
asm("multimem.st.release.cta.global.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__sem == sem_release && __scope == scope_cluster)
|
||||
{
|
||||
asm("multimem.st.release.cluster.global.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__sem == sem_release && __scope == scope_gpu)
|
||||
{
|
||||
asm("multimem.st.release.gpu.global.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__sem == sem_release && __scope == scope_sys)
|
||||
{
|
||||
asm("multimem.st.release.sys.global.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// multimem.st.sem.global.b64 [addr], val; // PTX ISA 81, SM_90
|
||||
// .sem = { .weak }
|
||||
template <typename B64, enable_if_t<sizeof(B64) == 8, bool> = true>
|
||||
__device__ static inline void multimem_st(
|
||||
cuda::ptx::sem_weak_t,
|
||||
B64* addr,
|
||||
B64 val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
|
||||
template <typename _B64, ::cuda::std::enable_if_t<sizeof(_B64) == 8, bool> = true>
|
||||
_CCCL_DEVICE static inline void multimem_st(::cuda::ptx::sem_weak_t, _B64* __addr, _B64 __val)
|
||||
{
|
||||
// __sem == sem_weak (due to parameter type constraint)
|
||||
static_assert(sizeof(_B64) == 8);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("multimem.st.weak.global.b64 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// multimem.st.sem.scope.global.b64 [addr], val; // PTX ISA 81, SM_90
|
||||
// .sem = { .relaxed, .release }
|
||||
// .scope = { .cta, .cluster, .gpu, .sys }
|
||||
template <typename B64, enable_if_t<sizeof(B64) == 8, bool> = true, cuda::ptx::dot_sem Sem, cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline void multimem_st(
|
||||
cuda::ptx::sem_t<Sem> sem,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
B64* addr,
|
||||
B64 val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
|
||||
template <typename _B64,
|
||||
::cuda::std::enable_if_t<sizeof(_B64) == 8, bool> = true,
|
||||
::cuda::ptx::dot_sem _Sem,
|
||||
::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline void
|
||||
multimem_st(::cuda::ptx::sem_t<_Sem> __sem, ::cuda::ptx::scope_t<_Scope> __scope, _B64* __addr, _B64 __val)
|
||||
{
|
||||
static_assert(__sem == sem_relaxed || __sem == sem_release);
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster || __scope == scope_gpu || __scope == scope_sys);
|
||||
static_assert(sizeof(_B64) == 8);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__sem == sem_relaxed && __scope == scope_cta)
|
||||
{
|
||||
asm("multimem.st.relaxed.cta.global.b64 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__sem == sem_relaxed && __scope == scope_cluster)
|
||||
{
|
||||
asm("multimem.st.relaxed.cluster.global.b64 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__sem == sem_relaxed && __scope == scope_gpu)
|
||||
{
|
||||
asm("multimem.st.relaxed.gpu.global.b64 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__sem == sem_relaxed && __scope == scope_sys)
|
||||
{
|
||||
asm("multimem.st.relaxed.sys.global.b64 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__sem == sem_release && __scope == scope_cta)
|
||||
{
|
||||
asm("multimem.st.release.cta.global.b64 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__sem == sem_release && __scope == scope_cluster)
|
||||
{
|
||||
asm("multimem.st.release.cluster.global.b64 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__sem == sem_release && __scope == scope_gpu)
|
||||
{
|
||||
asm("multimem.st.release.gpu.global.b64 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__sem == sem_release && __scope == scope_sys)
|
||||
{
|
||||
asm("multimem.st.release.sys.global.b64 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__addr)), "l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__val))
|
||||
: "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_MULTIMEM_ST_H_
|
||||
@@ -0,0 +1,230 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_PRMT_H_
|
||||
#define _CUDA_PTX_GENERATED_PRMT_H_
|
||||
|
||||
/*
|
||||
// prmt.b32 dest, a_reg, b_reg, c_reg; // PTX ISA 20, SM_50
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline uint32_t prmt(
|
||||
B32 a_reg,
|
||||
B32 b_reg,
|
||||
uint32_t c_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_prmt_is_not_supported_before_SM_50__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t prmt(_B32 __a_reg, _B32 __b_reg, ::cuda::std::uint32_t __c_reg)
|
||||
{
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("prmt.b32 %0, %1, %2, %3;"
|
||||
: "=r"(__dest)
|
||||
: "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__b_reg)),
|
||||
"r"(__c_reg)
|
||||
:);
|
||||
return __dest;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_prmt_is_not_supported_before_SM_50__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// prmt.b32.f4e dest, a_reg, b_reg, c_reg; // PTX ISA 20, SM_50
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline uint32_t prmt_f4e(
|
||||
B32 a_reg,
|
||||
B32 b_reg,
|
||||
uint32_t c_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_prmt_f4e_is_not_supported_before_SM_50__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t prmt_f4e(_B32 __a_reg, _B32 __b_reg, ::cuda::std::uint32_t __c_reg)
|
||||
{
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("prmt.b32.f4e %0, %1, %2, %3;"
|
||||
: "=r"(__dest)
|
||||
: "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__b_reg)),
|
||||
"r"(__c_reg)
|
||||
:);
|
||||
return __dest;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_prmt_f4e_is_not_supported_before_SM_50__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// prmt.b32.b4e dest, a_reg, b_reg, c_reg; // PTX ISA 20, SM_50
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline uint32_t prmt_b4e(
|
||||
B32 a_reg,
|
||||
B32 b_reg,
|
||||
uint32_t c_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_prmt_b4e_is_not_supported_before_SM_50__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t prmt_b4e(_B32 __a_reg, _B32 __b_reg, ::cuda::std::uint32_t __c_reg)
|
||||
{
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("prmt.b32.b4e %0, %1, %2, %3;"
|
||||
: "=r"(__dest)
|
||||
: "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__b_reg)),
|
||||
"r"(__c_reg)
|
||||
:);
|
||||
return __dest;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_prmt_b4e_is_not_supported_before_SM_50__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// prmt.b32.rc8 dest, a_reg, b_reg, c_reg; // PTX ISA 20, SM_50
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline uint32_t prmt_rc8(
|
||||
B32 a_reg,
|
||||
B32 b_reg,
|
||||
uint32_t c_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_prmt_rc8_is_not_supported_before_SM_50__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t prmt_rc8(_B32 __a_reg, _B32 __b_reg, ::cuda::std::uint32_t __c_reg)
|
||||
{
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("prmt.b32.rc8 %0, %1, %2, %3;"
|
||||
: "=r"(__dest)
|
||||
: "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__b_reg)),
|
||||
"r"(__c_reg)
|
||||
:);
|
||||
return __dest;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_prmt_rc8_is_not_supported_before_SM_50__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// prmt.b32.ecl dest, a_reg, b_reg, c_reg; // PTX ISA 20, SM_50
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline uint32_t prmt_ecl(
|
||||
B32 a_reg,
|
||||
B32 b_reg,
|
||||
uint32_t c_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_prmt_ecl_is_not_supported_before_SM_50__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t prmt_ecl(_B32 __a_reg, _B32 __b_reg, ::cuda::std::uint32_t __c_reg)
|
||||
{
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("prmt.b32.ecl %0, %1, %2, %3;"
|
||||
: "=r"(__dest)
|
||||
: "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__b_reg)),
|
||||
"r"(__c_reg)
|
||||
:);
|
||||
return __dest;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_prmt_ecl_is_not_supported_before_SM_50__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// prmt.b32.ecr dest, a_reg, b_reg, c_reg; // PTX ISA 20, SM_50
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline uint32_t prmt_ecr(
|
||||
B32 a_reg,
|
||||
B32 b_reg,
|
||||
uint32_t c_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_prmt_ecr_is_not_supported_before_SM_50__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t prmt_ecr(_B32 __a_reg, _B32 __b_reg, ::cuda::std::uint32_t __c_reg)
|
||||
{
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("prmt.b32.ecr %0, %1, %2, %3;"
|
||||
: "=r"(__dest)
|
||||
: "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__b_reg)),
|
||||
"r"(__c_reg)
|
||||
:);
|
||||
return __dest;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_prmt_ecr_is_not_supported_before_SM_50__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
/*
|
||||
// prmt.b32.rc16 dest, a_reg, b_reg, c_reg; // PTX ISA 20, SM_50
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline uint32_t prmt_rc16(
|
||||
B32 a_reg,
|
||||
B32 b_reg,
|
||||
uint32_t c_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 200
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_prmt_rc16_is_not_supported_before_SM_50__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline ::cuda::std::uint32_t prmt_rc16(_B32 __a_reg, _B32 __b_reg, ::cuda::std::uint32_t __c_reg)
|
||||
{
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("prmt.b32.rc16 %0, %1, %2, %3;"
|
||||
: "=r"(__dest)
|
||||
: "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__b_reg)),
|
||||
"r"(__c_reg)
|
||||
:);
|
||||
return __dest;
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_prmt_rc16_is_not_supported_before_SM_50__();
|
||||
return 0;
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 200
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_PRMT_H_
|
||||
@@ -0,0 +1,460 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_RED_ASYNC_H_
|
||||
#define _CUDA_PTX_GENERATED_RED_ASYNC_H_
|
||||
|
||||
/*
|
||||
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type [dest], value, [remote_bar]; // PTX
|
||||
ISA 81, SM_90
|
||||
// .type = { .u32 }
|
||||
// .op = { .inc }
|
||||
template <typename = void>
|
||||
__device__ static inline void red_async(
|
||||
cuda::ptx::op_inc_t,
|
||||
uint32_t* dest,
|
||||
const uint32_t& value,
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void
|
||||
red_async(::cuda::ptx::op_inc_t,
|
||||
::cuda::std::uint32_t* __dest,
|
||||
const ::cuda::std::uint32_t& __value,
|
||||
::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
// __type == type_u32 (due to parameter type constraint)
|
||||
// __op == op_inc (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.inc.u32 [%0], %1, [%2]; "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__dest)), "r"(__value), "r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type [dest], value, [remote_bar]; // PTX
|
||||
ISA 81, SM_90
|
||||
// .type = { .u32 }
|
||||
// .op = { .dec }
|
||||
template <typename = void>
|
||||
__device__ static inline void red_async(
|
||||
cuda::ptx::op_dec_t,
|
||||
uint32_t* dest,
|
||||
const uint32_t& value,
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void
|
||||
red_async(::cuda::ptx::op_dec_t,
|
||||
::cuda::std::uint32_t* __dest,
|
||||
const ::cuda::std::uint32_t& __value,
|
||||
::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
// __type == type_u32 (due to parameter type constraint)
|
||||
// __op == op_dec (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.dec.u32 [%0], %1, [%2]; "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__dest)), "r"(__value), "r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type [dest], value, [remote_bar]; // PTX
|
||||
ISA 81, SM_90
|
||||
// .type = { .u32 }
|
||||
// .op = { .min }
|
||||
template <typename = void>
|
||||
__device__ static inline void red_async(
|
||||
cuda::ptx::op_min_t,
|
||||
uint32_t* dest,
|
||||
const uint32_t& value,
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void
|
||||
red_async(::cuda::ptx::op_min_t,
|
||||
::cuda::std::uint32_t* __dest,
|
||||
const ::cuda::std::uint32_t& __value,
|
||||
::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
// __type == type_u32 (due to parameter type constraint)
|
||||
// __op == op_min (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.min.u32 [%0], %1, [%2]; "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__dest)), "r"(__value), "r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type [dest], value, [remote_bar]; // PTX
|
||||
ISA 81, SM_90
|
||||
// .type = { .u32 }
|
||||
// .op = { .max }
|
||||
template <typename = void>
|
||||
__device__ static inline void red_async(
|
||||
cuda::ptx::op_max_t,
|
||||
uint32_t* dest,
|
||||
const uint32_t& value,
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void
|
||||
red_async(::cuda::ptx::op_max_t,
|
||||
::cuda::std::uint32_t* __dest,
|
||||
const ::cuda::std::uint32_t& __value,
|
||||
::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
// __type == type_u32 (due to parameter type constraint)
|
||||
// __op == op_max (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.max.u32 [%0], %1, [%2]; "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__dest)), "r"(__value), "r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type [dest], value, [remote_bar]; // PTX
|
||||
ISA 81, SM_90
|
||||
// .type = { .u32 }
|
||||
// .op = { .add }
|
||||
template <typename = void>
|
||||
__device__ static inline void red_async(
|
||||
cuda::ptx::op_add_t,
|
||||
uint32_t* dest,
|
||||
const uint32_t& value,
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void
|
||||
red_async(::cuda::ptx::op_add_t,
|
||||
::cuda::std::uint32_t* __dest,
|
||||
const ::cuda::std::uint32_t& __value,
|
||||
::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
// __type == type_u32 (due to parameter type constraint)
|
||||
// __op == op_add (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.add.u32 [%0], %1, [%2]; "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__dest)), "r"(__value), "r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type [dest], value, [remote_bar]; // PTX
|
||||
ISA 81, SM_90
|
||||
// .type = { .s32 }
|
||||
// .op = { .min }
|
||||
template <typename = void>
|
||||
__device__ static inline void red_async(
|
||||
cuda::ptx::op_min_t,
|
||||
int32_t* dest,
|
||||
const int32_t& value,
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void
|
||||
red_async(::cuda::ptx::op_min_t,
|
||||
::cuda::std::int32_t* __dest,
|
||||
const ::cuda::std::int32_t& __value,
|
||||
::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
// __type == type_s32 (due to parameter type constraint)
|
||||
// __op == op_min (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.min.s32 [%0], %1, [%2]; "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__dest)), "r"(__value), "r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type [dest], value, [remote_bar]; // PTX
|
||||
ISA 81, SM_90
|
||||
// .type = { .s32 }
|
||||
// .op = { .max }
|
||||
template <typename = void>
|
||||
__device__ static inline void red_async(
|
||||
cuda::ptx::op_max_t,
|
||||
int32_t* dest,
|
||||
const int32_t& value,
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void
|
||||
red_async(::cuda::ptx::op_max_t,
|
||||
::cuda::std::int32_t* __dest,
|
||||
const ::cuda::std::int32_t& __value,
|
||||
::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
// __type == type_s32 (due to parameter type constraint)
|
||||
// __op == op_max (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.max.s32 [%0], %1, [%2]; "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__dest)), "r"(__value), "r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type [dest], value, [remote_bar]; // PTX
|
||||
ISA 81, SM_90
|
||||
// .type = { .s32 }
|
||||
// .op = { .add }
|
||||
template <typename = void>
|
||||
__device__ static inline void red_async(
|
||||
cuda::ptx::op_add_t,
|
||||
int32_t* dest,
|
||||
const int32_t& value,
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void
|
||||
red_async(::cuda::ptx::op_add_t,
|
||||
::cuda::std::int32_t* __dest,
|
||||
const ::cuda::std::int32_t& __value,
|
||||
::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
// __type == type_s32 (due to parameter type constraint)
|
||||
// __op == op_add (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.add.s32 [%0], %1, [%2]; "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__dest)), "r"(__value), "r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type [dest], value, [remote_bar]; // PTX
|
||||
ISA 81, SM_90
|
||||
// .type = { .b32 }
|
||||
// .op = { .and }
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void red_async(
|
||||
cuda::ptx::op_and_op_t,
|
||||
B32* dest,
|
||||
const B32& value,
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void
|
||||
red_async(::cuda::ptx::op_and_op_t, _B32* __dest, const _B32& __value, ::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
// __type == type_b32 (due to parameter type constraint)
|
||||
// __op == op_and_op (due to parameter type constraint)
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.and.b32 [%0], %1, [%2]; "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__dest)),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__value)),
|
||||
"r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type [dest], value, [remote_bar]; // PTX
|
||||
ISA 81, SM_90
|
||||
// .type = { .b32 }
|
||||
// .op = { .or }
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void red_async(
|
||||
cuda::ptx::op_or_op_t,
|
||||
B32* dest,
|
||||
const B32& value,
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void
|
||||
red_async(::cuda::ptx::op_or_op_t, _B32* __dest, const _B32& __value, ::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
// __type == type_b32 (due to parameter type constraint)
|
||||
// __op == op_or_op (due to parameter type constraint)
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.or.b32 [%0], %1, [%2]; "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__dest)),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__value)),
|
||||
"r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type [dest], value, [remote_bar]; // PTX
|
||||
ISA 81, SM_90
|
||||
// .type = { .b32 }
|
||||
// .op = { .xor }
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void red_async(
|
||||
cuda::ptx::op_xor_op_t,
|
||||
B32* dest,
|
||||
const B32& value,
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void
|
||||
red_async(::cuda::ptx::op_xor_op_t, _B32* __dest, const _B32& __value, ::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
// __type == type_b32 (due to parameter type constraint)
|
||||
// __op == op_xor_op (due to parameter type constraint)
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.xor.b32 [%0], %1, [%2]; "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__dest)),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__value)),
|
||||
"r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type [dest], value, [remote_bar]; // PTX
|
||||
ISA 81, SM_90
|
||||
// .type = { .u64 }
|
||||
// .op = { .add }
|
||||
template <typename = void>
|
||||
__device__ static inline void red_async(
|
||||
cuda::ptx::op_add_t,
|
||||
uint64_t* dest,
|
||||
const uint64_t& value,
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void
|
||||
red_async(::cuda::ptx::op_add_t,
|
||||
::cuda::std::uint64_t* __dest,
|
||||
const ::cuda::std::uint64_t& __value,
|
||||
::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
// __type == type_u64 (due to parameter type constraint)
|
||||
// __op == op_add (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.add.u64 [%0], %1, [%2]; "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__dest)), "l"(__value), "r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.u64 [dest], value, [remote_bar]; // .u64
|
||||
intentional PTX ISA 81, SM_90
|
||||
// .op = { .add }
|
||||
template <typename = void>
|
||||
__device__ static inline void red_async(
|
||||
cuda::ptx::op_add_t,
|
||||
int64_t* dest,
|
||||
const int64_t& value,
|
||||
int64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void
|
||||
red_async(::cuda::ptx::op_add_t,
|
||||
::cuda::std::int64_t* __dest,
|
||||
const ::cuda::std::int64_t& __value,
|
||||
::cuda::std::int64_t* __remote_bar)
|
||||
{
|
||||
// __op == op_add (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.add.u64 [%0], %1, [%2]; // .u64 "
|
||||
"intentional"
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__dest)), "l"(__value), "r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_red_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_RED_ASYNC_H_
|
||||
@@ -0,0 +1,58 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_SETMAXNREG_H_
|
||||
#define _CUDA_PTX_GENERATED_SETMAXNREG_H_
|
||||
|
||||
/*
|
||||
// setmaxnreg.inc.sync.aligned.u32 imm_reg_count; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a,
|
||||
SM_110f, SM_120a, SM_120f, SM_121a, SM_121f template <int N32>
|
||||
__device__ static inline void setmaxnreg_inc(
|
||||
cuda::ptx::n32_t<N32> imm_reg_count);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_setmaxnreg_inc_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void setmaxnreg_inc(::cuda::ptx::n32_t<_N32> __imm_reg_count)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm volatile("setmaxnreg.inc.sync.aligned.u32 %0;" : : "n"(__imm_reg_count.value) :);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_setmaxnreg_inc_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
/*
|
||||
// setmaxnreg.dec.sync.aligned.u32 imm_reg_count; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a,
|
||||
SM_110f, SM_120a, SM_120f, SM_121a, SM_121f template <int N32>
|
||||
__device__ static inline void setmaxnreg_dec(
|
||||
cuda::ptx::n32_t<N32> imm_reg_count);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 800
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_setmaxnreg_dec_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void setmaxnreg_dec(::cuda::ptx::n32_t<_N32> __imm_reg_count)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm volatile("setmaxnreg.dec.sync.aligned.u32 %0;" : : "n"(__imm_reg_count.value) :);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_setmaxnreg_dec_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 800
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_SETMAXNREG_H_
|
||||
@@ -0,0 +1,96 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_SHL_H_
|
||||
#define _CUDA_PTX_GENERATED_SHL_H_
|
||||
|
||||
/*
|
||||
// shl.b16 dest, a_reg, b_reg; // PTX ISA 10, SM_50
|
||||
template <typename B16, enable_if_t<sizeof(B16) == 2, bool> = true>
|
||||
__device__ static inline B16 shl(
|
||||
B16 a_reg,
|
||||
uint32_t b_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 100
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_shl_is_not_supported_before_SM_50__();
|
||||
template <typename _B16, ::cuda::std::enable_if_t<sizeof(_B16) == 2, bool> = true>
|
||||
_CCCL_DEVICE static inline _B16 shl(_B16 __a_reg, ::cuda::std::uint32_t __b_reg)
|
||||
{
|
||||
static_assert(sizeof(_B16) == 2);
|
||||
static_assert(sizeof(_B16) == 2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
|
||||
::cuda::std::uint16_t __dest;
|
||||
asm("shl.b16 %0, %1, %2;"
|
||||
: "=h"(__dest)
|
||||
: "h"(/*as_b16*/ *reinterpret_cast<const ::cuda::std::int16_t*>(&__a_reg)), "r"(__b_reg)
|
||||
:);
|
||||
return *reinterpret_cast<_B16*>(&__dest);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_shl_is_not_supported_before_SM_50__();
|
||||
::cuda::std::uint16_t __err_out_var = 0;
|
||||
return *reinterpret_cast<_B16*>(&__err_out_var);
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 100
|
||||
|
||||
/*
|
||||
// shl.b32 dest, a_reg, b_reg; // PTX ISA 10, SM_50
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline B32 shl(
|
||||
B32 a_reg,
|
||||
uint32_t b_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 100
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_shl_is_not_supported_before_SM_50__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline _B32 shl(_B32 __a_reg, ::cuda::std::uint32_t __b_reg)
|
||||
{
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("shl.b32 %0, %1, %2;"
|
||||
: "=r"(__dest)
|
||||
: "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)), "r"(__b_reg)
|
||||
:);
|
||||
return *reinterpret_cast<_B32*>(&__dest);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_shl_is_not_supported_before_SM_50__();
|
||||
::cuda::std::uint32_t __err_out_var = 0;
|
||||
return *reinterpret_cast<_B32*>(&__err_out_var);
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 100
|
||||
|
||||
/*
|
||||
// shl.b64 dest, a_reg, b_reg; // PTX ISA 10, SM_50
|
||||
template <typename B64, enable_if_t<sizeof(B64) == 8, bool> = true>
|
||||
__device__ static inline B64 shl(
|
||||
B64 a_reg,
|
||||
uint32_t b_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 100
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_shl_is_not_supported_before_SM_50__();
|
||||
template <typename _B64, ::cuda::std::enable_if_t<sizeof(_B64) == 8, bool> = true>
|
||||
_CCCL_DEVICE static inline _B64 shl(_B64 __a_reg, ::cuda::std::uint32_t __b_reg)
|
||||
{
|
||||
static_assert(sizeof(_B64) == 8);
|
||||
static_assert(sizeof(_B64) == 8);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
|
||||
::cuda::std::uint64_t __dest;
|
||||
asm("shl.b64 %0, %1, %2;"
|
||||
: "=l"(__dest)
|
||||
: "l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__a_reg)), "r"(__b_reg)
|
||||
:);
|
||||
return *reinterpret_cast<_B64*>(&__dest);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_shl_is_not_supported_before_SM_50__();
|
||||
::cuda::std::uint64_t __err_out_var = 0;
|
||||
return *reinterpret_cast<_B64*>(&__err_out_var);
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 100
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_SHL_H_
|
||||
@@ -0,0 +1,135 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_SHR_H_
|
||||
#define _CUDA_PTX_GENERATED_SHR_H_
|
||||
|
||||
/*
|
||||
// shr.b16 dest, a_reg, b_reg; // PTX ISA 10, SM_50
|
||||
template <typename B16, enable_if_t<sizeof(B16) == 2, bool> = true>
|
||||
__device__ static inline B16 shr(
|
||||
B16 a_reg,
|
||||
uint32_t b_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 100
|
||||
template <typename _B16, ::cuda::std::enable_if_t<sizeof(_B16) == 2, bool> = true>
|
||||
_CCCL_DEVICE static inline _B16 shr(_B16 __a_reg, ::cuda::std::uint32_t __b_reg)
|
||||
{
|
||||
static_assert(sizeof(_B16) == 2);
|
||||
static_assert(sizeof(_B16) == 2);
|
||||
::cuda::std::uint16_t __dest;
|
||||
asm("shr.b16 %0, %1, %2;"
|
||||
: "=h"(__dest)
|
||||
: "h"(/*as_b16*/ *reinterpret_cast<const ::cuda::std::int16_t*>(&__a_reg)), "r"(__b_reg)
|
||||
:);
|
||||
return *reinterpret_cast<_B16*>(&__dest);
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 100
|
||||
|
||||
/*
|
||||
// shr.b32 dest, a_reg, b_reg; // PTX ISA 10, SM_50
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4 && !(is_integral_v<B32> && is_signed_v<B32>), bool> = true>
|
||||
__device__ static inline B32 shr(
|
||||
B32 a_reg,
|
||||
uint32_t b_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 100
|
||||
template <
|
||||
typename _B32,
|
||||
::cuda::std::enable_if_t<sizeof(_B32) == 4 && !(::cuda::std::is_integral_v<_B32> && ::cuda::std::is_signed_v<_B32>),
|
||||
bool> = true>
|
||||
_CCCL_DEVICE static inline _B32 shr(_B32 __a_reg, ::cuda::std::uint32_t __b_reg)
|
||||
{
|
||||
::cuda::std::uint32_t __dest;
|
||||
asm("shr.b32 %0, %1, %2;"
|
||||
: "=r"(__dest)
|
||||
: "r"(*reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)), "r"(__b_reg)
|
||||
:);
|
||||
return *reinterpret_cast<_B32*>(&__dest);
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 100
|
||||
|
||||
/*
|
||||
// shr.b64 dest, a_reg, b_reg; // PTX ISA 10, SM_50
|
||||
template <typename B64, enable_if_t<sizeof(B64) == 8 && !(is_integral_v<B64> && is_signed_v<B64>), bool> = true>
|
||||
__device__ static inline B64 shr(
|
||||
B64 a_reg,
|
||||
uint32_t b_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 100
|
||||
template <
|
||||
typename _B64,
|
||||
::cuda::std::enable_if_t<sizeof(_B64) == 8 && !(::cuda::std::is_integral_v<_B64> && ::cuda::std::is_signed_v<_B64>),
|
||||
bool> = true>
|
||||
_CCCL_DEVICE static inline _B64 shr(_B64 __a_reg, ::cuda::std::uint32_t __b_reg)
|
||||
{
|
||||
::cuda::std::uint64_t __dest;
|
||||
asm("shr.b64 %0, %1, %2;"
|
||||
: "=l"(__dest)
|
||||
: "l"(*reinterpret_cast<const ::cuda::std::int64_t*>(&__a_reg)), "r"(__b_reg)
|
||||
:);
|
||||
return *reinterpret_cast<_B64*>(&__dest);
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 100
|
||||
|
||||
/*
|
||||
// shr.s16 dest, a_reg, b_reg; // PTX ISA 10, SM_50
|
||||
template <typename = void>
|
||||
__device__ static inline int16_t shr(
|
||||
int16_t a_reg,
|
||||
uint32_t b_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 100
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline ::cuda::std::int16_t shr(::cuda::std::int16_t __a_reg, ::cuda::std::uint32_t __b_reg)
|
||||
{
|
||||
::cuda::std::int16_t __dest;
|
||||
asm("shr.s16 %0, %1, %2;" : "=h"(__dest) : "h"(__a_reg), "r"(__b_reg) :);
|
||||
return __dest;
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 100
|
||||
|
||||
/*
|
||||
// shr.s32 dest, a_reg, b_reg; // PTX ISA 10, SM_50
|
||||
template <typename S32, enable_if_t<sizeof(S32) == 4 && is_integral_v<S32> && is_signed_v<S32>, bool> = true>
|
||||
__device__ static inline S32 shr(
|
||||
S32 a_reg,
|
||||
uint32_t b_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 100
|
||||
template <typename _S32,
|
||||
::cuda::std::enable_if_t<sizeof(_S32) == 4 && ::cuda::std::is_integral_v<_S32>&& ::cuda::std::is_signed_v<_S32>,
|
||||
bool> = true>
|
||||
_CCCL_DEVICE static inline _S32 shr(_S32 __a_reg, ::cuda::std::uint32_t __b_reg)
|
||||
{
|
||||
::cuda::std::int32_t __dest;
|
||||
asm("shr.s32 %0, %1, %2;"
|
||||
: "=r"(__dest)
|
||||
: "r"(*reinterpret_cast<const ::cuda::std::int32_t*>(&__a_reg)), "r"(__b_reg)
|
||||
:);
|
||||
return *reinterpret_cast<_S32*>(&__dest);
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 100
|
||||
|
||||
/*
|
||||
// shr.s64 dest, a_reg, b_reg; // PTX ISA 10, SM_50
|
||||
template <typename S64, enable_if_t<sizeof(S64) == 8 && is_integral_v<S64> && is_signed_v<S64>, bool> = true>
|
||||
__device__ static inline S64 shr(
|
||||
S64 a_reg,
|
||||
uint32_t b_reg);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 100
|
||||
template <typename _S64,
|
||||
::cuda::std::enable_if_t<sizeof(_S64) == 8 && ::cuda::std::is_integral_v<_S64>&& ::cuda::std::is_signed_v<_S64>,
|
||||
bool> = true>
|
||||
_CCCL_DEVICE static inline _S64 shr(_S64 __a_reg, ::cuda::std::uint32_t __b_reg)
|
||||
{
|
||||
::cuda::std::int64_t __dest;
|
||||
asm("shr.s64 %0, %1, %2;"
|
||||
: "=l"(__dest)
|
||||
: "l"(*reinterpret_cast<const ::cuda::std::int64_t*>(&__a_reg)), "r"(__b_reg)
|
||||
:);
|
||||
return *reinterpret_cast<_S64*>(&__dest);
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 100
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_SHR_H_
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,123 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_ST_ASYNC_H_
|
||||
#define _CUDA_PTX_GENERATED_ST_ASYNC_H_
|
||||
|
||||
/*
|
||||
// st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.type [addr], value, [remote_bar]; // 1. PTX ISA 81,
|
||||
SM_90
|
||||
// .type = { .b32, .b64 }
|
||||
template <typename Type>
|
||||
__device__ static inline void st_async(
|
||||
Type* addr,
|
||||
const Type& value,
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_st_async_is_not_supported_before_SM_90__();
|
||||
template <typename _Type>
|
||||
_CCCL_DEVICE static inline void st_async(_Type* __addr, const _Type& __value, ::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
static_assert(sizeof(_Type) == 4 || sizeof(_Type) == 8);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (sizeof(_Type) == 4)
|
||||
{
|
||||
asm("st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.b32 [%0], %1, [%2]; // 1. "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__addr)),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__value)),
|
||||
"r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (sizeof(_Type) == 8)
|
||||
{
|
||||
asm("st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.b64 [%0], %1, [%2]; // 1. "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__addr)),
|
||||
"l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__value)),
|
||||
"r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_st_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v2.type [addr], value, [remote_bar]; // 2. PTX ISA 81,
|
||||
SM_90
|
||||
// .type = { .b32, .b64 }
|
||||
template <typename Type>
|
||||
__device__ static inline void st_async(
|
||||
Type* addr,
|
||||
const Type (&value)[2],
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_st_async_is_not_supported_before_SM_90__();
|
||||
template <typename _Type>
|
||||
_CCCL_DEVICE static inline void st_async(_Type* __addr, const _Type (&__value)[2], ::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
static_assert(sizeof(_Type) == 4 || sizeof(_Type) == 8);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (sizeof(_Type) == 4)
|
||||
{
|
||||
asm("st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v2.b32 [%0], {%1, %2}, [%3]; // 2. "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__addr)),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__value[0])),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__value[1])),
|
||||
"r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (sizeof(_Type) == 8)
|
||||
{
|
||||
asm("st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v2.b64 [%0], {%1, %2}, [%3]; // 2. "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__addr)),
|
||||
"l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__value[0])),
|
||||
"l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__value[1])),
|
||||
"r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_st_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
/*
|
||||
// st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 [addr], value, [remote_bar]; // 3. PTX ISA 81,
|
||||
SM_90 template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void st_async(
|
||||
B32* addr,
|
||||
const B32 (&value)[4],
|
||||
uint64_t* remote_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 810
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_st_async_is_not_supported_before_SM_90__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void st_async(_B32* __addr, const _B32 (&__value)[4], ::cuda::std::uint64_t* __remote_bar)
|
||||
{
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
asm("st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 [%0], {%1, %2, %3, %4}, [%5]; // 3. "
|
||||
:
|
||||
: "r"(__as_ptr_remote_dsmem(__addr)),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__value[0])),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__value[1])),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__value[2])),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__value[3])),
|
||||
"r"(__as_ptr_remote_dsmem(__remote_bar))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_st_async_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 810
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_ST_ASYNC_H_
|
||||
@@ -0,0 +1,31 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_ST_BULK_H_
|
||||
#define _CUDA_PTX_GENERATED_ST_BULK_H_
|
||||
|
||||
/*
|
||||
// st.bulk.weak.shared::cta [addr], size, initval; // PTX ISA 86, SM_100
|
||||
template <int N32>
|
||||
__device__ static inline void st_bulk(
|
||||
void* addr,
|
||||
uint64_t size,
|
||||
cuda::ptx::n32_t<N32> initval);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_st_bulk_is_not_supported_before_SM_100__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void st_bulk(void* __addr, ::cuda::std::uint64_t __size, ::cuda::ptx::n32_t<_N32> __initval)
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 1000
|
||||
asm("st.bulk.weak.shared::cta [%0], %1, %2;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__addr)), "l"(__size), "n"(__initval.value)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_st_bulk_is_not_supported_before_SM_100__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_ST_BULK_H_
|
||||
@@ -0,0 +1,120 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_TCGEN05_ALLOC_H_
|
||||
#define _CUDA_PTX_GENERATED_TCGEN05_ALLOC_H_
|
||||
|
||||
/*
|
||||
// tcgen05.alloc.cta_group.sync.aligned.shared::cta.b32 [dst], nCols; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f,
|
||||
SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_alloc(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t* dst,
|
||||
const uint32_t& nCols);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_alloc_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_alloc(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t* __dst, const ::cuda::std::uint32_t& __nCols)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm volatile("tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dst)), "r"(__nCols)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm volatile("tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [%0], %1;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__dst)), "r"(__nCols)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_alloc_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.dealloc.cta_group.sync.aligned.b32 taddr, nCols; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a,
|
||||
SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_dealloc(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
const uint32_t& nCols);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_dealloc_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_dealloc(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, const ::cuda::std::uint32_t& __nCols)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm volatile("tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" : : "r"(__taddr), "r"(__nCols) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm volatile("tcgen05.dealloc.cta_group::2.sync.aligned.b32 %0, %1;" : : "r"(__taddr), "r"(__nCols) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_dealloc_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.relinquish_alloc_permit.cta_group.sync.aligned; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a,
|
||||
SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_relinquish_alloc_permit(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_relinquish_alloc_permit_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_relinquish_alloc_permit(::cuda::ptx::cta_group_t<_Cta_Group> __cta_group)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;" : : : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm volatile("tcgen05.relinquish_alloc_permit.cta_group::2.sync.aligned;" : : : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_relinquish_alloc_permit_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_TCGEN05_ALLOC_H_
|
||||
@@ -0,0 +1,91 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_TCGEN05_COMMIT_H_
|
||||
#define _CUDA_PTX_GENERATED_TCGEN05_COMMIT_H_
|
||||
|
||||
/*
|
||||
// tcgen05.commit.cta_group.mbarrier::arrive::one.shared::cluster.b64 [smem_bar]; // PTX ISA 86, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_commit(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint64_t* smem_bar);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_commit_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void
|
||||
tcgen05_commit(::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint64_t* __smem_bar)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm volatile("tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%0];"
|
||||
:
|
||||
: "r"(__as_ptr_dsmem(__smem_bar))
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm volatile("tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.b64 [%0];"
|
||||
:
|
||||
: "r"(__as_ptr_dsmem(__smem_bar))
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_commit_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.commit.cta_group.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [smem_bar], ctaMask; // PTX ISA
|
||||
86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_commit_multicast(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint64_t* smem_bar,
|
||||
uint16_t ctaMask);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_commit_multicast_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_commit_multicast(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint64_t* __smem_bar, ::cuda::std::uint16_t __ctaMask)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm volatile("tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%0], %1;"
|
||||
:
|
||||
: "r"(__as_ptr_dsmem(__smem_bar)), "h"(__ctaMask)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm volatile("tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%0], %1;"
|
||||
:
|
||||
: "r"(__as_ptr_dsmem(__smem_bar)), "h"(__ctaMask)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_commit_multicast_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_TCGEN05_COMMIT_H_
|
||||
@@ -0,0 +1,693 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_TCGEN05_CP_H_
|
||||
#define _CUDA_PTX_GENERATED_TCGEN05_CP_H_
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.128x256b [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_128x256b(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_128x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_128x256b(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.128x256b [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.128x256b [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_128x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.4x256b [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_4x256b(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_4x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_4x256b(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.4x256b [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.4x256b [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_4x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.128x128b [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_128x128b(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_128x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_128x128b(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.128x128b [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.128x128b [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_128x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.64x128b.warpx2::02_13 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f,
|
||||
SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_64x128b_warpx2_02_13(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_64x128b_warpx2_02_13_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_64x128b_warpx2_02_13(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.64x128b.warpx2::02_13 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.64x128b.warpx2::02_13 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_64x128b_warpx2_02_13_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.64x128b.warpx2::01_23 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f,
|
||||
SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_64x128b_warpx2_01_23(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_64x128b_warpx2_01_23_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_64x128b_warpx2_01_23(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.64x128b.warpx2::01_23 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.64x128b.warpx2::01_23 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_64x128b_warpx2_01_23_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.32x128b.warpx4 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a,
|
||||
SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_32x128b_warpx4(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_32x128b_warpx4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_32x128b_warpx4(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.32x128b.warpx4 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.32x128b.warpx4 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_32x128b_warpx4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.128x256b.b8x16.b6x16_p32 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f,
|
||||
SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_128x256b_b8x16_b6x16_p32(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_128x256b_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_128x256b_b8x16_b6x16_p32(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.128x256b.b8x16.b6x16_p32 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.128x256b.b8x16.b6x16_p32 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_128x256b_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.4x256b.b8x16.b6x16_p32 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f,
|
||||
SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_4x256b_b8x16_b6x16_p32(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_4x256b_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_4x256b_b8x16_b6x16_p32(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.4x256b.b8x16.b6x16_p32 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.4x256b.b8x16.b6x16_p32 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_4x256b_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.128x128b.b8x16.b6x16_p32 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f,
|
||||
SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_128x128b_b8x16_b6x16_p32(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_128x128b_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_128x128b_b8x16_b6x16_p32(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.128x128b.b8x16.b6x16_p32 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.128x128b.b8x16.b6x16_p32 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_128x128b_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.64x128b.warpx2::02_13.b8x16.b6x16_p32 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a,
|
||||
SM_103f, SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_64x128b_warpx2_02_13_b8x16_b6x16_p32(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_64x128b_warpx2_02_13_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_64x128b_warpx2_02_13_b8x16_b6x16_p32(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.64x128b.warpx2::02_13.b8x16.b6x16_p32 [%0], %1;"
|
||||
:
|
||||
: "r"(__taddr), "l"(__s_desc)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.64x128b.warpx2::02_13.b8x16.b6x16_p32 [%0], %1;"
|
||||
:
|
||||
: "r"(__taddr), "l"(__s_desc)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_64x128b_warpx2_02_13_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.64x128b.warpx2::01_23.b8x16.b6x16_p32 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a,
|
||||
SM_103f, SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_64x128b_warpx2_01_23_b8x16_b6x16_p32(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_64x128b_warpx2_01_23_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_64x128b_warpx2_01_23_b8x16_b6x16_p32(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.64x128b.warpx2::01_23.b8x16.b6x16_p32 [%0], %1;"
|
||||
:
|
||||
: "r"(__taddr), "l"(__s_desc)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.64x128b.warpx2::01_23.b8x16.b6x16_p32 [%0], %1;"
|
||||
:
|
||||
: "r"(__taddr), "l"(__s_desc)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_64x128b_warpx2_01_23_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.32x128b.warpx4.b8x16.b6x16_p32 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a,
|
||||
SM_103f, SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_32x128b_warpx4_b8x16_b6x16_p32(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_32x128b_warpx4_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_32x128b_warpx4_b8x16_b6x16_p32(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.32x128b.warpx4.b8x16.b6x16_p32 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.32x128b.warpx4.b8x16.b6x16_p32 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_32x128b_warpx4_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.128x256b.b8x16.b4x16_p64 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f,
|
||||
SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_128x256b_b8x16_b4x16_p64(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_128x256b_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_128x256b_b8x16_b4x16_p64(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.128x256b.b8x16.b4x16_p64 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.128x256b.b8x16.b4x16_p64 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_128x256b_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.4x256b.b8x16.b4x16_p64 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f,
|
||||
SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_4x256b_b8x16_b4x16_p64(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_4x256b_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_4x256b_b8x16_b4x16_p64(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.4x256b.b8x16.b4x16_p64 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.4x256b.b8x16.b4x16_p64 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_4x256b_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.128x128b.b8x16.b4x16_p64 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f,
|
||||
SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_128x128b_b8x16_b4x16_p64(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_128x128b_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_128x128b_b8x16_b4x16_p64(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.128x128b.b8x16.b4x16_p64 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.128x128b.b8x16.b4x16_p64 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_128x128b_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.64x128b.warpx2::02_13.b8x16.b4x16_p64 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a,
|
||||
SM_103f, SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_64x128b_warpx2_02_13_b8x16_b4x16_p64(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_64x128b_warpx2_02_13_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_64x128b_warpx2_02_13_b8x16_b4x16_p64(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.64x128b.warpx2::02_13.b8x16.b4x16_p64 [%0], %1;"
|
||||
:
|
||||
: "r"(__taddr), "l"(__s_desc)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.64x128b.warpx2::02_13.b8x16.b4x16_p64 [%0], %1;"
|
||||
:
|
||||
: "r"(__taddr), "l"(__s_desc)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_64x128b_warpx2_02_13_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.64x128b.warpx2::01_23.b8x16.b4x16_p64 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a,
|
||||
SM_103f, SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_64x128b_warpx2_01_23_b8x16_b4x16_p64(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_64x128b_warpx2_01_23_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_64x128b_warpx2_01_23_b8x16_b4x16_p64(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.64x128b.warpx2::01_23.b8x16.b4x16_p64 [%0], %1;"
|
||||
:
|
||||
: "r"(__taddr), "l"(__s_desc)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.64x128b.warpx2::01_23.b8x16.b4x16_p64 [%0], %1;"
|
||||
:
|
||||
: "r"(__taddr), "l"(__s_desc)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_64x128b_warpx2_01_23_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.cp.cta_group.32x128b.warpx4.b8x16.b4x16_p64 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a,
|
||||
SM_103f, SM_110a, SM_110f
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_cp_32x128b_warpx4_b8x16_b4x16_p64(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr,
|
||||
uint64_t s_desc);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_cp_32x128b_warpx4_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void tcgen05_cp_32x128b_warpx4_b8x16_b4x16_p64(
|
||||
::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr, ::cuda::std::uint64_t __s_desc)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::1.32x128b.warpx4.b8x16.b4x16_p64 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm("tcgen05.cp.cta_group::2.32x128b.warpx4.b8x16.b4x16_p64 [%0], %1;" : : "r"(__taddr), "l"(__s_desc) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_cp_32x128b_warpx4_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_TCGEN05_CP_H_
|
||||
@@ -0,0 +1,50 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_TCGEN05_FENCE_H_
|
||||
#define _CUDA_PTX_GENERATED_TCGEN05_FENCE_H_
|
||||
|
||||
/*
|
||||
// tcgen05.fence::before_thread_sync; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
template <typename = void>
|
||||
__device__ static inline void tcgen05_fence_before_thread_sync();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_fence_before_thread_sync_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void tcgen05_fence_before_thread_sync()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
asm volatile("tcgen05.fence::before_thread_sync;" : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_fence_before_thread_sync_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.fence::after_thread_sync; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
template <typename = void>
|
||||
__device__ static inline void tcgen05_fence_after_thread_sync();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_fence_after_thread_sync_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void tcgen05_fence_after_thread_sync()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
asm volatile("tcgen05.fence::after_thread_sync;" : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_fence_after_thread_sync_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_TCGEN05_FENCE_H_
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,40 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_TCGEN05_SHIFT_H_
|
||||
#define _CUDA_PTX_GENERATED_TCGEN05_SHIFT_H_
|
||||
|
||||
/*
|
||||
// tcgen05.shift.cta_group.down [taddr]; // PTX ISA 86, SM_100a, SM_103a, SM_110a
|
||||
// .cta_group = { .cta_group::1, .cta_group::2 }
|
||||
template <cuda::ptx::dot_cta_group Cta_Group>
|
||||
__device__ static inline void tcgen05_shift_down(
|
||||
cuda::ptx::cta_group_t<Cta_Group> cta_group,
|
||||
uint32_t taddr);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tcgen05_shift_down_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
|
||||
template <::cuda::ptx::dot_cta_group _Cta_Group>
|
||||
_CCCL_DEVICE static inline void
|
||||
tcgen05_shift_down(::cuda::ptx::cta_group_t<_Cta_Group> __cta_group, ::cuda::std::uint32_t __taddr)
|
||||
{
|
||||
static_assert(__cta_group == cta_group_1 || __cta_group == cta_group_2);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
|
||||
if constexpr (__cta_group == cta_group_1)
|
||||
{
|
||||
asm volatile("tcgen05.shift.cta_group::1.down [%0];" : : "r"(__taddr) : "memory");
|
||||
}
|
||||
else if constexpr (__cta_group == cta_group_2)
|
||||
{
|
||||
asm volatile("tcgen05.shift.cta_group::2.down [%0];" : : "r"(__taddr) : "memory");
|
||||
}
|
||||
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_shift_down_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_TCGEN05_SHIFT_H_
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,48 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_TCGEN05_WAIT_H_
|
||||
#define _CUDA_PTX_GENERATED_TCGEN05_WAIT_H_
|
||||
|
||||
/*
|
||||
// tcgen05.wait::ld.sync.aligned; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
template <typename = void>
|
||||
__device__ static inline void tcgen05_wait_ld();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_tcgen05_wait_ld_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void tcgen05_wait_ld()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
asm volatile("tcgen05.wait::ld.sync.aligned;" : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_wait_ld_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tcgen05.wait::st.sync.aligned; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
|
||||
template <typename = void>
|
||||
__device__ static inline void tcgen05_wait_st();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_tcgen05_wait_st_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void tcgen05_wait_st()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
|
||||
asm volatile("tcgen05.wait::st.sync.aligned;" : : : "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tcgen05_wait_st_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_TCGEN05_WAIT_H_
|
||||
@@ -0,0 +1,71 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_TENSORMAP_CP_FENCEPROXY_H_
|
||||
#define _CUDA_PTX_GENERATED_TENSORMAP_CP_FENCEPROXY_H_
|
||||
|
||||
/*
|
||||
// tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.sem.scope.sync.aligned [dst], [src], size; // PTX ISA
|
||||
83, SM_90
|
||||
// .sem = { .release }
|
||||
// .scope = { .cta, .cluster, .gpu, .sys }
|
||||
template <int N32, cuda::ptx::dot_scope Scope>
|
||||
__device__ static inline void tensormap_cp_fenceproxy(
|
||||
cuda::ptx::sem_release_t,
|
||||
cuda::ptx::scope_t<Scope> scope,
|
||||
void* dst,
|
||||
const void* src,
|
||||
cuda::ptx::n32_t<N32> size);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_tensormap_cp_fenceproxy_is_not_supported_before_SM_90__();
|
||||
template <int _N32, ::cuda::ptx::dot_scope _Scope>
|
||||
_CCCL_DEVICE static inline void tensormap_cp_fenceproxy(
|
||||
::cuda::ptx::sem_release_t,
|
||||
::cuda::ptx::scope_t<_Scope> __scope,
|
||||
void* __dst,
|
||||
const void* __src,
|
||||
::cuda::ptx::n32_t<_N32> __size)
|
||||
{
|
||||
// __sem == sem_release (due to parameter type constraint)
|
||||
static_assert(__scope == scope_cta || __scope == scope_cluster || __scope == scope_gpu || __scope == scope_sys);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
|
||||
if constexpr (__scope == scope_cta)
|
||||
{
|
||||
asm volatile(
|
||||
"tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.cta.sync.aligned [%0], [%1], %2;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__dst)), "r"(__as_ptr_smem(__src)), "n"(__size.value)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_cluster)
|
||||
{
|
||||
asm volatile(
|
||||
"tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.cluster.sync.aligned [%0], [%1], %2;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__dst)), "r"(__as_ptr_smem(__src)), "n"(__size.value)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_gpu)
|
||||
{
|
||||
asm volatile(
|
||||
"tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned [%0], [%1], %2;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__dst)), "r"(__as_ptr_smem(__src)), "n"(__size.value)
|
||||
: "memory");
|
||||
}
|
||||
else if constexpr (__scope == scope_sys)
|
||||
{
|
||||
asm volatile(
|
||||
"tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.sys.sync.aligned [%0], [%1], %2;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__dst)), "r"(__as_ptr_smem(__src)), "n"(__size.value)
|
||||
: "memory");
|
||||
}
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_cp_fenceproxy_is_not_supported_before_SM_90__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_TENSORMAP_CP_FENCEPROXY_H_
|
||||
@@ -0,0 +1,886 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_TENSORMAP_REPLACE_H_
|
||||
#define _CUDA_PTX_GENERATED_TENSORMAP_REPLACE_H_
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.global_address.space.b1024.b64 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .global }
|
||||
template <typename B64, enable_if_t<sizeof(B64) == 8, bool> = true>
|
||||
__device__ static inline void tensormap_replace_global_address(
|
||||
cuda::ptx::space_global_t,
|
||||
void* tm_addr,
|
||||
B64 new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_global_address_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <typename _B64, ::cuda::std::enable_if_t<sizeof(_B64) == 8, bool> = true>
|
||||
_CCCL_DEVICE static inline void
|
||||
tensormap_replace_global_address(::cuda::ptx::space_global_t, void* __tm_addr, _B64 __new_val)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(sizeof(_B64) == 8);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.global_address.global.b1024.b64 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__tm_addr)), "l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__new_val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_global_address_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.global_address.space.b1024.b64 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .shared::cta }
|
||||
template <typename B64, enable_if_t<sizeof(B64) == 8, bool> = true>
|
||||
__device__ static inline void tensormap_replace_global_address(
|
||||
cuda::ptx::space_shared_t,
|
||||
void* tm_addr,
|
||||
B64 new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_global_address_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <typename _B64, ::cuda::std::enable_if_t<sizeof(_B64) == 8, bool> = true>
|
||||
_CCCL_DEVICE static inline void
|
||||
tensormap_replace_global_address(::cuda::ptx::space_shared_t, void* __tm_addr, _B64 __new_val)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
static_assert(sizeof(_B64) == 8);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.global_address.shared::cta.b1024.b64 [%0], %1;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__tm_addr)), "l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__new_val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_global_address_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.rank.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a,
|
||||
SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .global }
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void tensormap_replace_rank(
|
||||
cuda::ptx::space_global_t,
|
||||
void* tm_addr,
|
||||
B32 new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_rank_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void tensormap_replace_rank(::cuda::ptx::space_global_t, void* __tm_addr, _B32 __new_val)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.rank.global.b1024.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__tm_addr)), "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__new_val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_rank_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.rank.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a,
|
||||
SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .shared::cta }
|
||||
template <typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void tensormap_replace_rank(
|
||||
cuda::ptx::space_shared_t,
|
||||
void* tm_addr,
|
||||
B32 new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_rank_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void tensormap_replace_rank(::cuda::ptx::space_shared_t, void* __tm_addr, _B32 __new_val)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.rank.shared::cta.b1024.b32 [%0], %1;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__tm_addr)), "r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__new_val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_rank_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.box_dim.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .global }
|
||||
template <int N32, typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void tensormap_replace_box_dim(
|
||||
cuda::ptx::space_global_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> ord,
|
||||
B32 new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_box_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32, typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void
|
||||
tensormap_replace_box_dim(::cuda::ptx::space_global_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __ord, _B32 __new_val)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.box_dim.global.b1024.b32 [%0], %1, %2;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__tm_addr)),
|
||||
"n"(__ord.value),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__new_val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_box_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.box_dim.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .shared::cta }
|
||||
template <int N32, typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void tensormap_replace_box_dim(
|
||||
cuda::ptx::space_shared_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> ord,
|
||||
B32 new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_box_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32, typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void
|
||||
tensormap_replace_box_dim(::cuda::ptx::space_shared_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __ord, _B32 __new_val)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.box_dim.shared::cta.b1024.b32 [%0], %1, %2;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__tm_addr)),
|
||||
"n"(__ord.value),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__new_val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_box_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.global_dim.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .global }
|
||||
template <int N32, typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void tensormap_replace_global_dim(
|
||||
cuda::ptx::space_global_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> ord,
|
||||
B32 new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_global_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32, typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void tensormap_replace_global_dim(
|
||||
::cuda::ptx::space_global_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __ord, _B32 __new_val)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.global_dim.global.b1024.b32 [%0], %1, %2;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__tm_addr)),
|
||||
"n"(__ord.value),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__new_val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_global_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.global_dim.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .shared::cta }
|
||||
template <int N32, typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void tensormap_replace_global_dim(
|
||||
cuda::ptx::space_shared_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> ord,
|
||||
B32 new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_global_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32, typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void tensormap_replace_global_dim(
|
||||
::cuda::ptx::space_shared_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __ord, _B32 __new_val)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.global_dim.shared::cta.b1024.b32 [%0], %1, %2;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__tm_addr)),
|
||||
"n"(__ord.value),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__new_val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_global_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.global_stride.space.b1024.b64 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a,
|
||||
SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .global }
|
||||
template <int N32, typename B64, enable_if_t<sizeof(B64) == 8, bool> = true>
|
||||
__device__ static inline void tensormap_replace_global_stride(
|
||||
cuda::ptx::space_global_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> ord,
|
||||
B64 new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_global_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32, typename _B64, ::cuda::std::enable_if_t<sizeof(_B64) == 8, bool> = true>
|
||||
_CCCL_DEVICE static inline void tensormap_replace_global_stride(
|
||||
::cuda::ptx::space_global_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __ord, _B64 __new_val)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(sizeof(_B64) == 8);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.global_stride.global.b1024.b64 [%0], %1, %2;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__tm_addr)),
|
||||
"n"(__ord.value),
|
||||
"l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__new_val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_global_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.global_stride.space.b1024.b64 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a,
|
||||
SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .shared::cta }
|
||||
template <int N32, typename B64, enable_if_t<sizeof(B64) == 8, bool> = true>
|
||||
__device__ static inline void tensormap_replace_global_stride(
|
||||
cuda::ptx::space_shared_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> ord,
|
||||
B64 new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_global_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32, typename _B64, ::cuda::std::enable_if_t<sizeof(_B64) == 8, bool> = true>
|
||||
_CCCL_DEVICE static inline void tensormap_replace_global_stride(
|
||||
::cuda::ptx::space_shared_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __ord, _B64 __new_val)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
static_assert(sizeof(_B64) == 8);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.global_stride.shared::cta.b1024.b64 [%0], %1, %2;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__tm_addr)),
|
||||
"n"(__ord.value),
|
||||
"l"(/*as_b64*/ *reinterpret_cast<const ::cuda::std::int64_t*>(&__new_val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_global_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.element_stride.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a,
|
||||
SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .global }
|
||||
template <int N32, typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void tensormap_replace_element_stride(
|
||||
cuda::ptx::space_global_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> ord,
|
||||
B32 new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_element_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32, typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void tensormap_replace_element_stride(
|
||||
::cuda::ptx::space_global_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __ord, _B32 __new_val)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.element_stride.global.b1024.b32 [%0], %1, %2;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__tm_addr)),
|
||||
"n"(__ord.value),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__new_val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_element_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.element_stride.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a,
|
||||
SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .shared::cta }
|
||||
template <int N32, typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void tensormap_replace_element_stride(
|
||||
cuda::ptx::space_shared_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> ord,
|
||||
B32 new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_element_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32, typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void tensormap_replace_element_stride(
|
||||
::cuda::ptx::space_shared_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __ord, _B32 __new_val)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.element_stride.shared::cta.b1024.b32 [%0], %1, %2;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__tm_addr)),
|
||||
"n"(__ord.value),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__new_val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_element_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.element_stride.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a,
|
||||
SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .global }
|
||||
template <int N32, typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void tensormap_replace_element_size(
|
||||
cuda::ptx::space_global_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> ord,
|
||||
B32 new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_element_size_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32, typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void tensormap_replace_element_size(
|
||||
::cuda::ptx::space_global_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __ord, _B32 __new_val)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.element_stride.global.b1024.b32 [%0], %1, %2;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__tm_addr)),
|
||||
"n"(__ord.value),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__new_val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_element_size_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.element_stride.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a,
|
||||
SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .shared::cta }
|
||||
template <int N32, typename B32, enable_if_t<sizeof(B32) == 4, bool> = true>
|
||||
__device__ static inline void tensormap_replace_element_size(
|
||||
cuda::ptx::space_shared_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> ord,
|
||||
B32 new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_element_size_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32, typename _B32, ::cuda::std::enable_if_t<sizeof(_B32) == 4, bool> = true>
|
||||
_CCCL_DEVICE static inline void tensormap_replace_element_size(
|
||||
::cuda::ptx::space_shared_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __ord, _B32 __new_val)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
static_assert(sizeof(_B32) == 4);
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.element_stride.shared::cta.b1024.b32 [%0], %1, %2;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__tm_addr)),
|
||||
"n"(__ord.value),
|
||||
"r"(/*as_b32*/ *reinterpret_cast<const ::cuda::std::int32_t*>(&__new_val))
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_element_size_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.elemtype.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a,
|
||||
SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .global }
|
||||
template <int N32>
|
||||
__device__ static inline void tensormap_replace_elemtype(
|
||||
cuda::ptx::space_global_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_elemtype_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void
|
||||
tensormap_replace_elemtype(::cuda::ptx::space_global_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __new_val)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.elemtype.global.b1024.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__tm_addr)), "n"(__new_val.value)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_elemtype_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.elemtype.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a,
|
||||
SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .shared::cta }
|
||||
template <int N32>
|
||||
__device__ static inline void tensormap_replace_elemtype(
|
||||
cuda::ptx::space_shared_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_elemtype_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void
|
||||
tensormap_replace_elemtype(::cuda::ptx::space_shared_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __new_val)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.elemtype.shared::cta.b1024.b32 [%0], %1;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__tm_addr)), "n"(__new_val.value)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_elemtype_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.interleave_layout.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .global }
|
||||
template <int N32>
|
||||
__device__ static inline void tensormap_replace_interleave_layout(
|
||||
cuda::ptx::space_global_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_interleave_layout_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void
|
||||
tensormap_replace_interleave_layout(::cuda::ptx::space_global_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __new_val)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.interleave_layout.global.b1024.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__tm_addr)), "n"(__new_val.value)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_interleave_layout_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.interleave_layout.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .shared::cta }
|
||||
template <int N32>
|
||||
__device__ static inline void tensormap_replace_interleave_layout(
|
||||
cuda::ptx::space_shared_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_interleave_layout_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void
|
||||
tensormap_replace_interleave_layout(::cuda::ptx::space_shared_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __new_val)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.interleave_layout.shared::cta.b1024.b32 [%0], %1;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__tm_addr)), "n"(__new_val.value)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_interleave_layout_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.swizzle_mode.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .global }
|
||||
template <int N32>
|
||||
__device__ static inline void tensormap_replace_swizzle_mode(
|
||||
cuda::ptx::space_global_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_swizzle_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void
|
||||
tensormap_replace_swizzle_mode(::cuda::ptx::space_global_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __new_val)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.swizzle_mode.global.b1024.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__tm_addr)), "n"(__new_val.value)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_swizzle_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.swizzle_mode.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .shared::cta }
|
||||
template <int N32>
|
||||
__device__ static inline void tensormap_replace_swizzle_mode(
|
||||
cuda::ptx::space_shared_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_swizzle_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void
|
||||
tensormap_replace_swizzle_mode(::cuda::ptx::space_shared_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __new_val)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.swizzle_mode.shared::cta.b1024.b32 [%0], %1;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__tm_addr)), "n"(__new_val.value)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_swizzle_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.fill_mode.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .global }
|
||||
template <int N32>
|
||||
__device__ static inline void tensormap_replace_fill_mode(
|
||||
cuda::ptx::space_global_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_fill_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void
|
||||
tensormap_replace_fill_mode(::cuda::ptx::space_global_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __new_val)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.fill_mode.global.b1024.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__tm_addr)), "n"(__new_val.value)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_fill_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.fill_mode.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .shared::cta }
|
||||
template <int N32>
|
||||
__device__ static inline void tensormap_replace_fill_mode(
|
||||
cuda::ptx::space_shared_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 830
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_fill_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void
|
||||
tensormap_replace_fill_mode(::cuda::ptx::space_shared_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __new_val)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.fill_mode.shared::cta.b1024.b32 [%0], %1;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__tm_addr)), "n"(__new_val.value)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_fill_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 830
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.swizzle_atomicity.space.b1024.b32 [tm_addr], new_val; // PTX ISA 86, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .global }
|
||||
template <int N32>
|
||||
__device__ static inline void tensormap_replace_swizzle_atomicity(
|
||||
cuda::ptx::space_global_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_swizzle_atomicity_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void
|
||||
tensormap_replace_swizzle_atomicity(::cuda::ptx::space_global_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __new_val)
|
||||
{
|
||||
// __space == space_global (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.swizzle_atomicity.global.b1024.b32 [%0], %1;"
|
||||
:
|
||||
: "l"(__as_ptr_gmem(__tm_addr)), "n"(__new_val.value)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_swizzle_atomicity_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
/*
|
||||
// tensormap.replace.tile.swizzle_atomicity.space.b1024.b32 [tm_addr], new_val; // PTX ISA 86, SM_100a, SM_100f,
|
||||
SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
|
||||
// .space = { .shared::cta }
|
||||
template <int N32>
|
||||
__device__ static inline void tensormap_replace_swizzle_atomicity(
|
||||
cuda::ptx::space_shared_t,
|
||||
void* tm_addr,
|
||||
cuda::ptx::n32_t<N32> new_val);
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 860
|
||||
extern "C" _CCCL_DEVICE void
|
||||
__cuda_ptx_tensormap_replace_swizzle_atomicity_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
template <int _N32>
|
||||
_CCCL_DEVICE static inline void
|
||||
tensormap_replace_swizzle_atomicity(::cuda::ptx::space_shared_t, void* __tm_addr, ::cuda::ptx::n32_t<_N32> __new_val)
|
||||
{
|
||||
// __space == space_shared (due to parameter type constraint)
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) \
|
||||
|| (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) \
|
||||
|| __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
|
||||
asm("tensormap.replace.tile.swizzle_atomicity.shared::cta.b1024.b32 [%0], %1;"
|
||||
:
|
||||
: "r"(__as_ptr_smem(__tm_addr)), "n"(__new_val.value)
|
||||
: "memory");
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_tensormap_replace_swizzle_atomicity_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 860
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_TENSORMAP_REPLACE_H_
|
||||
@@ -0,0 +1,25 @@
|
||||
// This file was automatically generated. Do not edit.
|
||||
|
||||
#ifndef _CUDA_PTX_GENERATED_TRAP_H_
|
||||
#define _CUDA_PTX_GENERATED_TRAP_H_
|
||||
|
||||
/*
|
||||
// trap; // PTX ISA 10, SM_50
|
||||
template <typename = void>
|
||||
__device__ static inline void trap();
|
||||
*/
|
||||
#if __cccl_ptx_isa >= 100
|
||||
extern "C" _CCCL_DEVICE void __cuda_ptx_trap_is_not_supported_before_SM_50__();
|
||||
template <typename = void>
|
||||
_CCCL_DEVICE static inline void trap()
|
||||
{
|
||||
# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
|
||||
asm volatile("trap;" : : :);
|
||||
# else
|
||||
// Unsupported architectures will have a linker error with a semi-decent error message
|
||||
__cuda_ptx_trap_is_not_supported_before_SM_50__();
|
||||
# endif
|
||||
}
|
||||
#endif // __cccl_ptx_isa >= 100
|
||||
|
||||
#endif // _CUDA_PTX_GENERATED_TRAP_H_
|
||||
@@ -0,0 +1,43 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_GET_SREG_H_
|
||||
#define _CUDA_PTX_GET_SREG_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 10. Special Registers
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#special-registers
|
||||
#include <cuda/__ptx/instructions/generated/get_sreg.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_GET_SREG_H_
|
||||
@@ -0,0 +1,43 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_GETCTARANK_H_
|
||||
#define _CUDA_PTX_GETCTARANK_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 9.7.8.23. Data Movement and Conversion Instructions: getctarank
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-getctarank
|
||||
#include <cuda/__ptx/instructions/generated/getctarank.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_GETCTARANK_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_LD_H_
|
||||
#define _CUDA_PTX_LD_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/ld.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_LD_H_
|
||||
@@ -0,0 +1,45 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_MBARRIER_ARRIVE_H_
|
||||
#define _CUDA_PTX_MBARRIER_ARRIVE_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 9.7.12.15.13. Parallel Synchronization and Communication Instructions: mbarrier.arrive
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive
|
||||
#include <cuda/__ptx/instructions/generated/mbarrier_arrive.h>
|
||||
#include <cuda/__ptx/instructions/generated/mbarrier_arrive_expect_tx.h>
|
||||
#include <cuda/__ptx/instructions/generated/mbarrier_arrive_no_complete.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_MBARRIER_ARRIVE_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_MBARRIER_EXPECT_TX_H_
|
||||
#define _CUDA_PTX_MBARRIER_EXPECT_TX_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/mbarrier_expect_tx.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_MBARRIER_EXPECT_TX_H_
|
||||
@@ -0,0 +1,43 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_MBARRIER_INIT_H_
|
||||
#define _CUDA_PTX_MBARRIER_INIT_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 9.7.12.15.9. Parallel Synchronization and Communication Instructions: mbarrier.init
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init
|
||||
#include <cuda/__ptx/instructions/generated/mbarrier_init.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_MBARRIER_INIT_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_MBARRIER_INVAL_H_
|
||||
#define _CUDA_PTX_MBARRIER_INVAL_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/mbarrier_inval.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_MBARRIER_INVAL_H_
|
||||
@@ -0,0 +1,46 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_MBARRIER_WAIT_H_
|
||||
#define _CUDA_PTX_MBARRIER_WAIT_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 9.7.12.15.16. Parallel Synchronization and Communication Instructions: mbarrier.test_wait/mbarrier.try_wait
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait
|
||||
#include <cuda/__ptx/instructions/generated/mbarrier_test_wait.h>
|
||||
#include <cuda/__ptx/instructions/generated/mbarrier_test_wait_parity.h>
|
||||
#include <cuda/__ptx/instructions/generated/mbarrier_try_wait.h>
|
||||
#include <cuda/__ptx/instructions/generated/mbarrier_try_wait_parity.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_MBARRIER_WAIT_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_MULTIMEM_LD_REDUCE_H_
|
||||
#define _CUDA_PTX_MULTIMEM_LD_REDUCE_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/multimem_ld_reduce.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_MULTIMEM_LD_REDUCE_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_MULTIMEM_RED_H_
|
||||
#define _CUDA_PTX_MULTIMEM_RED_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/multimem_red.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_MULTIMEM_RED_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_MULTIMEM_ST_H_
|
||||
#define _CUDA_PTX_MULTIMEM_ST_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/multimem_st.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_MULTIMEM_ST_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_PRMT_H_
|
||||
#define _CUDA_PTX_PRMT_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/prmt.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_PRMT_H_
|
||||
@@ -0,0 +1,43 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_RED_ASYNC_H_
|
||||
#define _CUDA_PTX_RED_ASYNC_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 9.7.12.7. Parallel Synchronization and Communication Instructions: red.async
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-red-async
|
||||
#include <cuda/__ptx/instructions/generated/red_async.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_RED_ASYNC_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_SETMAXNREG_H_
|
||||
#define _CUDA_PTX_SETMAXNREG_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/setmaxnreg.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_SETMAXNREG_H_
|
||||
@@ -0,0 +1,244 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_SHFL_SYNC_H
|
||||
#define _CUDA_PTX_SHFL_SYNC_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/instructions/get_sreg.h>
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/std/__bit/bit_cast.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#if __cccl_ptx_isa >= 600
|
||||
|
||||
enum class __dot_shfl_mode
|
||||
{
|
||||
__up,
|
||||
__down,
|
||||
__bfly,
|
||||
__idx
|
||||
};
|
||||
|
||||
[[maybe_unused]]
|
||||
_CCCL_DEVICE static inline uint32_t
|
||||
__shfl_sync_dst_lane(__dot_shfl_mode __shfl_mode, uint32_t __lane_idx_offset, uint32_t __clamp_segmask)
|
||||
{
|
||||
auto __lane = ::cuda::ptx::get_sreg_laneid();
|
||||
auto __clamp = __clamp_segmask & 0b11111;
|
||||
auto __segmask = __clamp_segmask >> 8;
|
||||
auto __max_lane = (__lane & __segmask) | (__clamp & ~__segmask);
|
||||
uint32_t __j = 0;
|
||||
if (__shfl_mode == __dot_shfl_mode::__idx)
|
||||
{
|
||||
auto __min_lane = __lane & __segmask;
|
||||
__j = __min_lane | (__lane_idx_offset & ~__segmask);
|
||||
}
|
||||
else if (__shfl_mode == __dot_shfl_mode::__up)
|
||||
{
|
||||
__j = __lane_idx_offset >= __lane ? 0 : __lane - __lane_idx_offset;
|
||||
}
|
||||
else if (__shfl_mode == __dot_shfl_mode::__down)
|
||||
{
|
||||
__j = __lane + __lane_idx_offset;
|
||||
}
|
||||
else
|
||||
{
|
||||
__j = __lane ^ __lane_idx_offset;
|
||||
}
|
||||
auto __dst = __shfl_mode == __dot_shfl_mode::__up
|
||||
? (__j >= __max_lane ? __j : __lane) //
|
||||
: (__j <= __max_lane ? __j : __lane);
|
||||
return (1u << __dst);
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_CCCL_DEVICE static inline void __shfl_sync_checks(
|
||||
__dot_shfl_mode __shfl_mode,
|
||||
_Tp,
|
||||
[[maybe_unused]] uint32_t __lane_idx_offset,
|
||||
[[maybe_unused]] uint32_t __clamp_segmask,
|
||||
[[maybe_unused]] uint32_t __lane_mask)
|
||||
{
|
||||
static_assert(sizeof(_Tp) == 4, "shfl.sync only accepts 4-byte data types");
|
||||
_CCCL_ASSERT(__lane_mask & (1u << ::cuda::ptx::get_sreg_laneid()), "lane_mask must contain the current lane");
|
||||
if (__shfl_mode != __dot_shfl_mode::__idx)
|
||||
{
|
||||
_CCCL_ASSERT(__lane_idx_offset < 32, "the lane index or offset must be less than the warp size");
|
||||
}
|
||||
_CCCL_ASSERT((__clamp_segmask | 0b1111100011111) == 0b1111100011111,
|
||||
"clamp value + segmentation mask must use the bit positions [0:4] and [8:12]");
|
||||
_CCCL_ASSERT(::cuda::ptx::__shfl_sync_dst_lane(__shfl_mode, __lane_idx_offset, __clamp_segmask) & __lane_mask,
|
||||
"the destination lane must be a member of the lane mask");
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
[[nodiscard]] _CCCL_DEVICE static inline _Tp shfl_sync_idx(
|
||||
_Tp __data, bool& __pred, uint32_t __lane_idx_offset, uint32_t __clamp_segmask, uint32_t __lane_mask) noexcept
|
||||
{
|
||||
::cuda::ptx::__shfl_sync_checks(__dot_shfl_mode::__idx, __data, __lane_idx_offset, __clamp_segmask, __lane_mask);
|
||||
auto __data1 = ::cuda::std::bit_cast<uint32_t>(__data);
|
||||
int __pred1;
|
||||
uint32_t __ret;
|
||||
asm volatile(
|
||||
"{ \n\t\t"
|
||||
".reg .pred p; \n\t\t"
|
||||
"shfl.sync.idx.b32 %0|p, %2, %3, %4, %5; \n\t\t"
|
||||
"selp.s32 %1, 1, 0, p; \n\t"
|
||||
"}"
|
||||
: "=r"(__ret), "=r"(__pred1)
|
||||
: "r"(__data1), "r"(__lane_idx_offset), "r"(__clamp_segmask), "r"(__lane_mask));
|
||||
__pred = static_cast<bool>(__pred1);
|
||||
return ::cuda::std::bit_cast<_Tp>(__ret);
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
[[nodiscard]] _CCCL_DEVICE static inline _Tp
|
||||
shfl_sync_idx(_Tp __data, uint32_t __lane_idx_offset, uint32_t __clamp_segmask, uint32_t __lane_mask) noexcept
|
||||
{
|
||||
::cuda::ptx::__shfl_sync_checks(__dot_shfl_mode::__idx, __data, __lane_idx_offset, __clamp_segmask, __lane_mask);
|
||||
auto __data1 = ::cuda::std::bit_cast<uint32_t>(__data);
|
||||
uint32_t __ret;
|
||||
asm volatile("{ \n\t\t"
|
||||
"shfl.sync.idx.b32 %0, %1, %2, %3, %4; \n\t\t"
|
||||
"}"
|
||||
: "=r"(__ret)
|
||||
: "r"(__data1), "r"(__lane_idx_offset), "r"(__clamp_segmask), "r"(__lane_mask));
|
||||
return ::cuda::std::bit_cast<_Tp>(__ret);
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
[[nodiscard]] _CCCL_DEVICE static inline _Tp shfl_sync_up(
|
||||
_Tp __data, bool& __pred, uint32_t __lane_idx_offset, uint32_t __clamp_segmask, uint32_t __lane_mask) noexcept
|
||||
{
|
||||
::cuda::ptx::__shfl_sync_checks(__dot_shfl_mode::__up, __data, __lane_idx_offset, __clamp_segmask, __lane_mask);
|
||||
auto __data1 = ::cuda::std::bit_cast<uint32_t>(__data);
|
||||
int __pred1;
|
||||
uint32_t __ret;
|
||||
asm volatile(
|
||||
"{ \n\t\t"
|
||||
".reg .pred p; \n\t\t"
|
||||
"shfl.sync.up.b32 %0|p, %2, %3, %4, %5; \n\t\t"
|
||||
"selp.s32 %1, 1, 0, p; \n\t"
|
||||
"}"
|
||||
: "=r"(__ret), "=r"(__pred1)
|
||||
: "r"(__data1), "r"(__lane_idx_offset), "r"(__clamp_segmask), "r"(__lane_mask));
|
||||
__pred = static_cast<bool>(__pred1);
|
||||
return ::cuda::std::bit_cast<_Tp>(__ret);
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
[[nodiscard]] _CCCL_DEVICE static inline _Tp
|
||||
shfl_sync_up(_Tp __data, uint32_t __lane_idx_offset, uint32_t __clamp_segmask, uint32_t __lane_mask) noexcept
|
||||
{
|
||||
::cuda::ptx::__shfl_sync_checks(__dot_shfl_mode::__up, __data, __lane_idx_offset, __clamp_segmask, __lane_mask);
|
||||
auto __data1 = ::cuda::std::bit_cast<uint32_t>(__data);
|
||||
uint32_t __ret;
|
||||
asm volatile("{ \n\t\t"
|
||||
"shfl.sync.up.b32 %0, %1, %2, %3, %4; \n\t\t"
|
||||
"}"
|
||||
: "=r"(__ret)
|
||||
: "r"(__data1), "r"(__lane_idx_offset), "r"(__clamp_segmask), "r"(__lane_mask));
|
||||
return ::cuda::std::bit_cast<_Tp>(__ret);
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
[[nodiscard]] _CCCL_DEVICE static inline _Tp shfl_sync_down(
|
||||
_Tp __data, bool& __pred, uint32_t __lane_idx_offset, uint32_t __clamp_segmask, uint32_t __lane_mask) noexcept
|
||||
{
|
||||
::cuda::ptx::__shfl_sync_checks(__dot_shfl_mode::__down, __data, __lane_idx_offset, __clamp_segmask, __lane_mask);
|
||||
auto __data1 = ::cuda::std::bit_cast<uint32_t>(__data);
|
||||
int __pred1;
|
||||
uint32_t __ret;
|
||||
asm volatile(
|
||||
"{ \n\t\t"
|
||||
".reg .pred p; \n\t\t"
|
||||
"shfl.sync.down.b32 %0|p, %2, %3, %4, %5; \n\t\t"
|
||||
"selp.s32 %1, 1, 0, p; \n\t"
|
||||
"}"
|
||||
: "=r"(__ret), "=r"(__pred1)
|
||||
: "r"(__data1), "r"(__lane_idx_offset), "r"(__clamp_segmask), "r"(__lane_mask));
|
||||
__pred = static_cast<bool>(__pred1);
|
||||
return ::cuda::std::bit_cast<_Tp>(__ret);
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
[[nodiscard]] _CCCL_DEVICE static inline _Tp
|
||||
shfl_sync_down(_Tp __data, uint32_t __lane_idx_offset, uint32_t __clamp_segmask, uint32_t __lane_mask) noexcept
|
||||
{
|
||||
::cuda::ptx::__shfl_sync_checks(__dot_shfl_mode::__down, __data, __lane_idx_offset, __clamp_segmask, __lane_mask);
|
||||
auto __data1 = ::cuda::std::bit_cast<uint32_t>(__data);
|
||||
uint32_t __ret;
|
||||
asm volatile("{ \n\t\t"
|
||||
"shfl.sync.down.b32 %0, %1, %2, %3, %4; \n\t\t"
|
||||
"}"
|
||||
: "=r"(__ret)
|
||||
: "r"(__data1), "r"(__lane_idx_offset), "r"(__clamp_segmask), "r"(__lane_mask));
|
||||
return ::cuda::std::bit_cast<_Tp>(__ret);
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
[[nodiscard]] _CCCL_DEVICE static inline _Tp shfl_sync_bfly(
|
||||
_Tp __data, bool& __pred, uint32_t __lane_idx_offset, uint32_t __clamp_segmask, uint32_t __lane_mask) noexcept
|
||||
{
|
||||
::cuda::ptx::__shfl_sync_checks(__dot_shfl_mode::__bfly, __data, __lane_idx_offset, __clamp_segmask, __lane_mask);
|
||||
auto __data1 = ::cuda::std::bit_cast<uint32_t>(__data);
|
||||
int __pred1;
|
||||
uint32_t __ret;
|
||||
asm volatile(
|
||||
"{ \n\t\t"
|
||||
".reg .pred p; \n\t\t"
|
||||
"shfl.sync.bfly.b32 %0|p, %2, %3, %4, %5; \n\t\t"
|
||||
"selp.s32 %1, 1, 0, p; \n\t"
|
||||
"}"
|
||||
: "=r"(__ret), "=r"(__pred1)
|
||||
: "r"(__data1), "r"(__lane_idx_offset), "r"(__clamp_segmask), "r"(__lane_mask));
|
||||
__pred = static_cast<bool>(__pred1);
|
||||
return ::cuda::std::bit_cast<_Tp>(__ret);
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
[[nodiscard]] _CCCL_DEVICE static inline _Tp
|
||||
shfl_sync_bfly(_Tp __data, uint32_t __lane_idx_offset, uint32_t __clamp_segmask, uint32_t __lane_mask) noexcept
|
||||
{
|
||||
::cuda::ptx::__shfl_sync_checks(__dot_shfl_mode::__bfly, __data, __lane_idx_offset, __clamp_segmask, __lane_mask);
|
||||
auto __data1 = ::cuda::std::bit_cast<uint32_t>(__data);
|
||||
uint32_t __ret;
|
||||
asm volatile( //
|
||||
"{ \n\t\t"
|
||||
"shfl.sync.bfly.b32 %0, %1, %2, %3, %4; \n\t\t"
|
||||
"}"
|
||||
: "=r"(__ret)
|
||||
: "r"(__data1), "r"(__lane_idx_offset), "r"(__clamp_segmask), "r"(__lane_mask));
|
||||
return ::cuda::std::bit_cast<_Tp>(__ret);
|
||||
}
|
||||
|
||||
#endif // __cccl_ptx_isa >= 600
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_SHFL_SYNC_H
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_SHL_H_
|
||||
#define _CUDA_PTX_SHL_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/shl.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_SHL_H_
|
||||
@@ -0,0 +1,43 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_SHR_H_
|
||||
#define _CUDA_PTX_SHR_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/__type_traits/is_integral.h>
|
||||
#include <cuda/std/__type_traits/is_signed.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/shr.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_SHR_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_ST_H_
|
||||
#define _CUDA_PTX_ST_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/st.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_ST_H_
|
||||
@@ -0,0 +1,43 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_ST_ASYNC_H_
|
||||
#define _CUDA_PTX_ST_ASYNC_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
// 9.7.8.12. Data Movement and Conversion Instructions: st.async
|
||||
// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-st-async
|
||||
#include <cuda/__ptx/instructions/generated/st_async.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_ST_ASYNC_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_ST_BULK_H_
|
||||
#define _CUDA_PTX_ST_BULK_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/st_bulk.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_ST_BULK_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_TCGEN05_ALLOC_H_
|
||||
#define _CUDA_PTX_TCGEN05_ALLOC_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/tcgen05_alloc.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_TCGEN05_ALLOC_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_TCGEN05_COMMIT_H_
|
||||
#define _CUDA_PTX_TCGEN05_COMMIT_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/tcgen05_commit.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_TCGEN05_COMMIT_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
// -*- C++ -*-
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_PTX_TCGEN05_CP_H_
|
||||
#define _CUDA_PTX_TCGEN05_CP_H_
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/ptx_dot_variants.h>
|
||||
#include <cuda/__ptx/ptx_helper_functions.h>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/__ptx/instructions/generated/tcgen05_cp.h>
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_PTX
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_PTX_TCGEN05_CP_H_
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user