[CCCL] 瘦身 + 补全: 移除 cudax/python/libcudacxx-tests 冗余文件, 新增 c2h 测试助手 + cmake 构建系统 + 8 个 CUDA thrust examples
变更摘要:
- 删除: cudax/ (783 files, 7.2M) — 实验性组件，竞赛不需要
- 删除: python/ (226 files, 2.0M) — Python 绑定，竞赛不需要
- 删除: libcudacxx/{test,benchmarks,codegen,cmake,share} (4432 files, 31M)
保留: libcudacxx/include/ (1463 headers, cuda::std 编译依赖)
- 新增: c2h/ (27 files) — CUB Catch2 测试辅助头文件，编译 243 个测试必需
- 新增: cmake/ (29 files) — CCCL 原生 CMake 构建系统
- 新增: thrust/examples/cuda/ (7 files) + cpp_integration/ (1 file)
async_reduce, custom_temporary_allocation, explicit_cuda_stream,
global_device_vector, range_view, unwrap_pointer, wrap_pointer, device
结果: cccl_upstream 从 74M→35M (瘦身 53%), 核心内容 100% 保留:
27/27 tuning headers, 78 benchmarks, 243 tests,
60 thrust examples, 18 CUB examples, 全部编译头文件
This commit is contained in:
muh-bot
@@ -1,339 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <thrust/host_vector.h>
|
||||
|
||||
#include <cuda/std/array>
|
||||
|
||||
#include "copy_common.cuh"
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* 1D Tests
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (16):(1)
|
||||
// dst: (16):(1)
|
||||
TEST_CASE("copy d2d 1D", "[copy][d2d][1d]")
|
||||
{
|
||||
constexpr int N = 16;
|
||||
test_copy<layout_right>(make_iota<int>(N), N);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* 2D Tests
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (4,8):(8,1)
|
||||
// dst: (4,8):(8,1)
|
||||
TEST_CASE("copy d2d 2D row-major to row-major", "[copy][d2d][2d][basic]")
|
||||
{
|
||||
constexpr int M = 4;
|
||||
constexpr int N = 8;
|
||||
test_copy<layout_right>(make_iota<int>(M * N), M, N);
|
||||
}
|
||||
|
||||
// src: (4,8):(1,4)
|
||||
// dst: (4,8):(1,4)
|
||||
TEST_CASE("copy d2d 2D column-major to column-major", "[copy][d2d][2d][basic]")
|
||||
{
|
||||
constexpr int M = 4;
|
||||
constexpr int N = 8;
|
||||
test_copy<layout_left>(make_iota<int>(M * N), M, N);
|
||||
}
|
||||
|
||||
// src: (4,8):(8,1)
|
||||
// dst: (4,8):(1,4)
|
||||
TEST_CASE("copy d2d 2D row-major to column-major", "[copy][d2d][2d][basic]")
|
||||
{
|
||||
constexpr int M = 4;
|
||||
constexpr int N = 8;
|
||||
auto data = make_iota<int>(M * N);
|
||||
thrust::host_vector<int> expected(M * N);
|
||||
for (int i = 0; i < M; ++i)
|
||||
{
|
||||
for (int j = 0; j < N; ++j)
|
||||
{
|
||||
expected[i + static_cast<std::size_t>(j) * M] = static_cast<int>(static_cast<std::size_t>(i) * N + j);
|
||||
}
|
||||
}
|
||||
test_copy<layout_right, layout_left>(data, expected, M, N);
|
||||
}
|
||||
|
||||
// src: (4,8):(1,4)
|
||||
// dst: (4,8):(8,1)
|
||||
TEST_CASE("copy d2d 2D column-major to row-major", "[copy][d2d][2d][basic]")
|
||||
{
|
||||
constexpr int M = 4;
|
||||
constexpr int N = 8;
|
||||
auto data = make_iota<int>(M * N);
|
||||
thrust::host_vector<int> expected(M * N);
|
||||
for (int i = 0; i < M; ++i)
|
||||
{
|
||||
for (int j = 0; j < N; ++j)
|
||||
{
|
||||
expected[static_cast<std::size_t>(i) * N + j] = static_cast<int>(i + static_cast<std::size_t>(j) * M);
|
||||
}
|
||||
}
|
||||
test_copy<layout_left, layout_right>(data, expected, M, N);
|
||||
}
|
||||
|
||||
// src: (1280,2564):(2564,1)
|
||||
// dst: (1280,2564):(2564,1)
|
||||
TEST_CASE("copy d2d 2D large", "[copy][d2d][2d][large]")
|
||||
{
|
||||
constexpr int M = 1280;
|
||||
constexpr int N = 2564;
|
||||
test_copy<layout_right>(make_iota<int>(M * N), M, N);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* 3D Tests
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (2,3,4):(12,4,1)
|
||||
// dst: (2,3,4):(12,4,1)
|
||||
TEST_CASE("copy d2d 3D row-major", "[copy][d2d][3d]")
|
||||
{
|
||||
constexpr int D0 = 2;
|
||||
constexpr int D1 = 3;
|
||||
constexpr int D2 = 4;
|
||||
test_copy<layout_right>(make_iota<int>(D0 * D1 * D2), D0, D1, D2);
|
||||
}
|
||||
|
||||
// src: (2,3,4):(12,4,1)
|
||||
// dst: (2,3,4):(1,2,6)
|
||||
TEST_CASE("copy d2d 3D row-major to column-major", "[copy][d2d][3d]")
|
||||
{
|
||||
constexpr int D0 = 2;
|
||||
constexpr int D1 = 3;
|
||||
constexpr int D2 = 4;
|
||||
constexpr int total = D0 * D1 * D2;
|
||||
auto data = make_iota<int>(total);
|
||||
thrust::host_vector<int> expected(total);
|
||||
for (int i = 0; i < D0; ++i)
|
||||
{
|
||||
for (int j = 0; j < D1; ++j)
|
||||
{
|
||||
for (int k = 0; k < D2; ++k)
|
||||
{
|
||||
expected[i + static_cast<std::size_t>(j) * D0 + static_cast<std::size_t>(k) * D0 * D1] =
|
||||
static_cast<int>(static_cast<std::size_t>(i) * (D1 * D2) + static_cast<std::size_t>(j) * D2 + k);
|
||||
}
|
||||
}
|
||||
}
|
||||
test_copy<layout_right, layout_left>(data, expected, D0, D1, D2);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Strided Layout Tests
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (4,8):(16,1)
|
||||
// dst: (4,8):(16,1)
|
||||
TEST_CASE("copy d2d 2D strided padded row-major", "[copy][d2d][2d][stride][row]")
|
||||
{
|
||||
constexpr int M = 4;
|
||||
constexpr int N = 8;
|
||||
constexpr int Ld = 16;
|
||||
cuda::std::array<int, 2> shape{M, N};
|
||||
cuda::std::array<int, 2> strides{Ld, 1};
|
||||
|
||||
using extents_t = cuda::std::dextents<int, 2>;
|
||||
using mapping_t = cuda::std::layout_stride::mapping<extents_t>;
|
||||
const auto span_size = static_cast<int>(mapping_t(extents_t(shape), strides).required_span_size());
|
||||
|
||||
thrust::host_vector<int> data(span_size, 0);
|
||||
for (int i = 0; i < M; ++i)
|
||||
{
|
||||
for (int j = 0; j < N; ++j)
|
||||
{
|
||||
data[static_cast<std::size_t>(i) * Ld + j] = static_cast<int>(static_cast<std::size_t>(i) * N + j);
|
||||
}
|
||||
}
|
||||
test_copy_strided(data, shape, strides);
|
||||
}
|
||||
|
||||
// src: (4,8):(1,16)
|
||||
// dst: (4,8):(1,16)
|
||||
TEST_CASE("copy d2d 2D strided padded column-major", "[copy][d2d][2d][stride][column]")
|
||||
{
|
||||
constexpr int M = 4;
|
||||
constexpr int N = 8;
|
||||
constexpr int Ld = 16;
|
||||
cuda::std::array<int, 2> shape{M, N};
|
||||
cuda::std::array<int, 2> strides{1, Ld};
|
||||
|
||||
using extents_t = cuda::std::dextents<int, 2>;
|
||||
using mapping_t = cuda::std::layout_stride::mapping<extents_t>;
|
||||
const auto span_size = static_cast<int>(mapping_t(extents_t(shape), strides).required_span_size());
|
||||
|
||||
thrust::host_vector<int> data(span_size, 0);
|
||||
for (int i = 0; i < M; ++i)
|
||||
{
|
||||
for (int j = 0; j < N; ++j)
|
||||
{
|
||||
data[i + static_cast<std::size_t>(j) * Ld] = static_cast<int>(static_cast<std::size_t>(i) * N + j);
|
||||
}
|
||||
}
|
||||
test_copy_strided(data, shape, strides);
|
||||
}
|
||||
|
||||
// src: (2,3,4):(12,4,1)
|
||||
// dst: (2,3,4):(1,8,2)
|
||||
TEST_CASE("copy d2d 3D strided permutation", "[copy][d2d][3d][stride][permutation]")
|
||||
{
|
||||
constexpr int D0 = 2;
|
||||
constexpr int D1 = 3;
|
||||
constexpr int D2 = 4;
|
||||
auto input = make_iota<int>(D0 * D1 * D2);
|
||||
cuda::std::array<int, 3> shape{D0, D1, D2};
|
||||
cuda::std::array<int, 3> src_strides{D1 * D2, D2, 1};
|
||||
cuda::std::array<int, 3> dst_strides{1, D2 * D0, D0};
|
||||
|
||||
using extents_t = cuda::std::dextents<int, 3>;
|
||||
using mapping_t = cuda::std::layout_stride::mapping<extents_t>;
|
||||
const auto dst_span = static_cast<int>(mapping_t(extents_t(shape), dst_strides).required_span_size());
|
||||
|
||||
thrust::host_vector<int> expected(dst_span, 0);
|
||||
for (int i = 0; i < D0; ++i)
|
||||
{
|
||||
for (int j = 0; j < D1; ++j)
|
||||
{
|
||||
for (int k = 0; k < D2; ++k)
|
||||
{
|
||||
expected[i + static_cast<std::size_t>(j) * (D2 * D0) + static_cast<std::size_t>(k) * D0] =
|
||||
input[static_cast<std::size_t>(i) * (D1 * D2) + static_cast<std::size_t>(j) * D2 + k];
|
||||
}
|
||||
}
|
||||
}
|
||||
test_copy_strided(input, expected, shape, src_strides, dst_strides);
|
||||
}
|
||||
|
||||
// src: (2,3,4):(12,4,1)
|
||||
// dst: (2,3,4):(8,16,1)
|
||||
TEST_CASE("copy d2d 3D strided different stride order", "[copy][d2d][3d][stride][tile]")
|
||||
{
|
||||
constexpr int D0 = 2;
|
||||
constexpr int D1 = 3;
|
||||
constexpr int D2 = 4;
|
||||
auto input = make_iota<int>(D0 * D1 * D2);
|
||||
cuda::std::array<int, 3> shape{D0, D1, D2};
|
||||
cuda::std::array<int, 3> src_strides{D1 * D2, D2, 1};
|
||||
cuda::std::array<int, 3> dst_strides{8, 16, 1};
|
||||
|
||||
using extents_t = cuda::std::dextents<int, 3>;
|
||||
using mapping_t = cuda::std::layout_stride::mapping<extents_t>;
|
||||
const auto dst_span = static_cast<int>(mapping_t(extents_t(shape), dst_strides).required_span_size());
|
||||
|
||||
thrust::host_vector<int> expected(dst_span, 0);
|
||||
for (int i = 0; i < D0; ++i)
|
||||
{
|
||||
for (int j = 0; j < D1; ++j)
|
||||
{
|
||||
for (int k = 0; k < D2; ++k)
|
||||
{
|
||||
expected[static_cast<std::size_t>(i) * 8 + static_cast<std::size_t>(j) * 16 + k] =
|
||||
input[static_cast<std::size_t>(i) * (D1 * D2) + static_cast<std::size_t>(j) * D2 + k];
|
||||
}
|
||||
}
|
||||
}
|
||||
test_copy_strided(input, expected, shape, src_strides, dst_strides);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Different Element Types
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (1024):(1)
|
||||
// dst: (1024):(1)
|
||||
TEST_CASE("copy d2d 1D double", "[copy][d2d][types][double]")
|
||||
{
|
||||
constexpr int N = 1024;
|
||||
thrust::host_vector<double> data(N);
|
||||
for (int i = 0; i < N; ++i)
|
||||
{
|
||||
data[i] = static_cast<double>(i) * 0.5;
|
||||
}
|
||||
test_copy<layout_right>(data, N);
|
||||
}
|
||||
|
||||
// src: (2048):(1)
|
||||
// dst: (2048):(1)
|
||||
TEST_CASE("copy d2d 1D short", "[copy][d2d][types][short]")
|
||||
{
|
||||
constexpr int N = 2048;
|
||||
thrust::host_vector<short> data(N);
|
||||
for (int i = 0; i < N; ++i)
|
||||
{
|
||||
data[i] = static_cast<short>(i % 1000);
|
||||
}
|
||||
test_copy<layout_right>(data, N);
|
||||
}
|
||||
|
||||
// src: (4096):(1)
|
||||
// dst: (4096):(1)
|
||||
TEST_CASE("copy d2d 1D char", "[copy][d2d][types][char]")
|
||||
{
|
||||
constexpr int N = 4096;
|
||||
thrust::host_vector<char> data(N);
|
||||
for (int i = 0; i < N; ++i)
|
||||
{
|
||||
data[i] = static_cast<char>(i % 128);
|
||||
}
|
||||
test_copy<layout_right>(data, N);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Large / Multi-Block Tests
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (100000):(1)
|
||||
// dst: (100000):(1)
|
||||
TEST_CASE("copy d2d 1D large", "[copy][d2d][1d][large]")
|
||||
{
|
||||
constexpr int N = 100000;
|
||||
test_copy<layout_right>(make_iota<float>(N), N);
|
||||
}
|
||||
|
||||
// src: (13,17):(17,1)
|
||||
// dst: (13,17):(1,13)
|
||||
TEST_CASE("copy d2d 2D transposition non-tile-divisible", "[copy][d2d][2d][boundary]")
|
||||
{
|
||||
constexpr int M = 13;
|
||||
constexpr int N = 17;
|
||||
auto data = make_iota<float>(M * N);
|
||||
thrust::host_vector<float> expected(M * N);
|
||||
for (int r = 0; r < M; ++r)
|
||||
{
|
||||
for (int c = 0; c < N; ++c)
|
||||
{
|
||||
expected[r + static_cast<std::size_t>(c) * M] = data[static_cast<std::size_t>(r) * N + c];
|
||||
}
|
||||
}
|
||||
test_copy<layout_right, layout_left>(data, expected, M, N);
|
||||
}
|
||||
|
||||
// src: (100,200):(200,1)
|
||||
// dst: (100,200):(1,100)
|
||||
TEST_CASE("copy d2d 2D large transposition", "[copy][d2d][2d][large][transpose]")
|
||||
{
|
||||
constexpr int M = 100;
|
||||
constexpr int N = 200;
|
||||
auto data = make_iota<int>(M * N);
|
||||
thrust::host_vector<int> expected(M * N);
|
||||
for (int r = 0; r < M; ++r)
|
||||
{
|
||||
for (int c = 0; c < N; ++c)
|
||||
{
|
||||
expected[r + static_cast<std::size_t>(c) * M] = data[static_cast<std::size_t>(r) * N + c];
|
||||
}
|
||||
}
|
||||
test_copy<layout_right, layout_left>(data, expected, M, N);
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef CUDAX_TEST_COPY_COMMON_CUH
|
||||
#define CUDAX_TEST_COPY_COMMON_CUH
|
||||
|
||||
#include <thrust/device_vector.h>
|
||||
#include <thrust/host_vector.h>
|
||||
|
||||
#include <cuda/mdspan>
|
||||
#include <cuda/std/array>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/copy.cuh>
|
||||
|
||||
#include "testing.cuh"
|
||||
|
||||
using cuda::std::layout_left;
|
||||
using cuda::std::layout_right;
|
||||
|
||||
static const cuda::stream stream{cuda::device_ref{0}};
|
||||
|
||||
template <typename T>
|
||||
thrust::host_vector<T> make_iota(int n)
|
||||
{
|
||||
thrust::host_vector<T> data(n);
|
||||
for (int i = 0; i < n; ++i)
|
||||
{
|
||||
data[i] = static_cast<T>(i);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------------------------------------------------
|
||||
// Basic Layout Tests
|
||||
|
||||
template <typename SrcLayout, typename DstLayout, typename T, typename... Ints>
|
||||
void test_copy(const thrust::host_vector<T>& input, const thrust::host_vector<T>& expected, Ints... shape)
|
||||
{
|
||||
[[maybe_unused]] constexpr size_t Rank = sizeof...(Ints); // msvc warns, only used in nttp
|
||||
using extents_t = cuda::std::dextents<int, Rank>;
|
||||
extents_t ext(static_cast<int>(shape)...);
|
||||
typename SrcLayout::template mapping<extents_t> src_mapping(ext);
|
||||
typename DstLayout::template mapping<extents_t> dst_mapping(ext);
|
||||
|
||||
thrust::device_vector<T> d_src(input.begin(), input.end());
|
||||
thrust::device_vector<T> d_dst(expected.size(), T{0});
|
||||
|
||||
cuda::device_mdspan<T, extents_t, SrcLayout> src(thrust::raw_pointer_cast(d_src.data()), src_mapping);
|
||||
cuda::device_mdspan<T, extents_t, DstLayout> dst(thrust::raw_pointer_cast(d_dst.data()), dst_mapping);
|
||||
|
||||
cuda::experimental::copy(src, dst, stream);
|
||||
stream.sync();
|
||||
|
||||
thrust::host_vector<T> result(d_dst);
|
||||
REQUIRE(result == expected);
|
||||
}
|
||||
|
||||
template <typename Layout, typename T, typename... Ints>
|
||||
void test_copy(const thrust::host_vector<T>& data, Ints... shape)
|
||||
{
|
||||
test_copy<Layout, Layout>(data, data, shape...);
|
||||
}
|
||||
|
||||
template <typename T, typename... Ints>
|
||||
void test_copy_iota(Ints... shape)
|
||||
{
|
||||
test_copy<layout_right>(make_iota<T>((static_cast<int>(shape) * ...)), shape...);
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------------------------------------------------
|
||||
// Strided Layout Tests
|
||||
|
||||
template <typename T, size_t Rank>
|
||||
void test_copy_strided(
|
||||
const thrust::host_vector<T>& input,
|
||||
const thrust::host_vector<T>& expected,
|
||||
const cuda::std::array<int, Rank>& shape,
|
||||
const cuda::std::array<int, Rank>& src_strides,
|
||||
const cuda::std::array<int, Rank>& dst_strides)
|
||||
{
|
||||
using extents_t = cuda::std::dextents<int, Rank>;
|
||||
using mapping_t = cuda::std::layout_stride::mapping<extents_t>;
|
||||
extents_t ext(shape);
|
||||
mapping_t src_mapping(ext, src_strides);
|
||||
mapping_t dst_mapping(ext, dst_strides);
|
||||
|
||||
thrust::device_vector<T> d_src(input.begin(), input.end());
|
||||
thrust::device_vector<T> d_dst(expected.size(), T{0});
|
||||
|
||||
cuda::device_mdspan<T, extents_t, cuda::std::layout_stride> src(thrust::raw_pointer_cast(d_src.data()), src_mapping);
|
||||
cuda::device_mdspan<T, extents_t, cuda::std::layout_stride> dst(thrust::raw_pointer_cast(d_dst.data()), dst_mapping);
|
||||
|
||||
cuda::experimental::copy(src, dst, stream);
|
||||
stream.sync();
|
||||
|
||||
thrust::host_vector<T> result(d_dst);
|
||||
REQUIRE(result == expected);
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------------------------------------------------
|
||||
// Strided Relaxed Layout Tests
|
||||
|
||||
template <typename T, size_t Rank>
|
||||
void test_copy_strided(const thrust::host_vector<T>& data,
|
||||
const cuda::std::array<int, Rank>& shape,
|
||||
const cuda::std::array<int, Rank>& strides)
|
||||
{
|
||||
test_copy_strided(data, data, shape, strides, strides);
|
||||
}
|
||||
|
||||
template <typename T, size_t Rank>
|
||||
void test_copy_stride_relaxed(
|
||||
int src_alloc,
|
||||
int src_offset,
|
||||
const cuda::std::array<int, Rank>& shape,
|
||||
const cuda::std::array<int, Rank>& src_strides,
|
||||
int dst_alloc,
|
||||
int dst_offset,
|
||||
const cuda::std::array<int, Rank>& dst_strides)
|
||||
{
|
||||
auto h_src = make_iota<T>(src_alloc);
|
||||
int total = 1;
|
||||
for (size_t r = 0; r < Rank; ++r)
|
||||
{
|
||||
total *= shape[r];
|
||||
}
|
||||
thrust::host_vector<T> expected(dst_alloc, T{0});
|
||||
for (int flat = 0; flat < total; ++flat)
|
||||
{
|
||||
cuda::std::array<int, Rank> idx{};
|
||||
for (int r = int{Rank} - 1, tmp = flat; r >= 0; --r)
|
||||
{
|
||||
idx[r] = tmp % shape[r];
|
||||
tmp /= shape[r];
|
||||
}
|
||||
int src_linear = src_offset;
|
||||
int dst_linear = dst_offset;
|
||||
for (size_t r = 0; r < Rank; ++r)
|
||||
{
|
||||
src_linear += idx[r] * src_strides[r];
|
||||
dst_linear += idx[r] * dst_strides[r];
|
||||
}
|
||||
expected[dst_linear] = h_src[src_linear];
|
||||
}
|
||||
thrust::device_vector<T> d_src(h_src.begin(), h_src.end());
|
||||
thrust::device_vector<T> d_dst(dst_alloc, T{0});
|
||||
|
||||
using extents_t = cuda::std::dextents<int, Rank>;
|
||||
using strides_t = cuda::dstrides<int, Rank>;
|
||||
using mapping_t = cuda::layout_stride_relaxed::mapping<extents_t>;
|
||||
|
||||
extents_t ext(shape);
|
||||
auto src_ptr = thrust::raw_pointer_cast(d_src.data());
|
||||
auto dst_ptr = thrust::raw_pointer_cast(d_dst.data());
|
||||
mapping_t src_map(ext, strides_t(src_strides), src_offset);
|
||||
mapping_t dst_map(ext, strides_t(dst_strides), dst_offset);
|
||||
|
||||
cuda::device_mdspan<T, extents_t, cuda::layout_stride_relaxed> src(src_ptr, src_map);
|
||||
cuda::device_mdspan<T, extents_t, cuda::layout_stride_relaxed> dst(dst_ptr, dst_map);
|
||||
|
||||
cuda::experimental::copy(src, dst, stream);
|
||||
stream.sync();
|
||||
|
||||
thrust::host_vector<T> result(d_dst);
|
||||
REQUIRE(result == expected);
|
||||
}
|
||||
|
||||
template <typename T, size_t Rank>
|
||||
void test_copy_stride_relaxed(
|
||||
int alloc, int offset, const cuda::std::array<int, Rank>& shape, const cuda::std::array<int, Rank>& strides)
|
||||
{
|
||||
test_copy_stride_relaxed<T>(alloc, offset, shape, strides, alloc, offset, strides);
|
||||
}
|
||||
|
||||
#endif // CUDAX_TEST_COPY_COMMON_CUH
|
||||
@@ -1,463 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <thrust/device_vector.h>
|
||||
#include <thrust/host_vector.h>
|
||||
|
||||
#include <cuda/std/climits>
|
||||
#include <cuda/std/linalg>
|
||||
|
||||
#include <stdexcept>
|
||||
|
||||
#include "copy_common.cuh"
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Edge Cases
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (1):(1)
|
||||
// dst: (1):(1)
|
||||
TEST_CASE("copy d2d scalar", "[copy][d2d][0d]")
|
||||
{
|
||||
thrust::host_vector<int> data(1, 42);
|
||||
test_copy<layout_right>(data, 1);
|
||||
}
|
||||
|
||||
// src: int (8):(1)
|
||||
// dst: float (8):(1)
|
||||
// __to_raw_tensor removes singleton dims, so we use N > 1 to avoid rank-0 tensors.
|
||||
// __are_byte_copyable is false when types differ → bypasses memcpy fast path.
|
||||
// __have_default_accessors is true, tile_size == tensor_size → path (1) DeviceTransform
|
||||
TEST_CASE("copy d2d different types", "[copy][d2d][1d][mixed_types]")
|
||||
{
|
||||
namespace cudax = cuda::experimental;
|
||||
constexpr int N = 8;
|
||||
thrust::host_vector<int> h_src(N);
|
||||
for (int i = 0; i < N; ++i)
|
||||
{
|
||||
h_src[i] = i * 10;
|
||||
}
|
||||
thrust::device_vector<int> d_src(h_src.begin(), h_src.end());
|
||||
thrust::device_vector<float> d_dst(N, 0.0f);
|
||||
|
||||
using extents_t = cuda::std::dextents<int, 1>;
|
||||
extents_t ext(N);
|
||||
layout_right::mapping<extents_t> mapping(ext);
|
||||
|
||||
cuda::device_mdspan<int, extents_t, layout_right> src(thrust::raw_pointer_cast(d_src.data()), mapping);
|
||||
cuda::device_mdspan<float, extents_t, layout_right> dst(thrust::raw_pointer_cast(d_dst.data()), mapping);
|
||||
|
||||
cudax::copy(src, dst, stream);
|
||||
stream.sync();
|
||||
|
||||
thrust::host_vector<float> h_expected(N);
|
||||
for (int i = 0; i < N; ++i)
|
||||
{
|
||||
h_expected[i] = static_cast<float>(i * 10);
|
||||
}
|
||||
thrust::host_vector<float> result(d_dst);
|
||||
REQUIRE(result == h_expected);
|
||||
}
|
||||
|
||||
// src: (0,0):(0,1)
|
||||
// dst: (0,0):(0,1)
|
||||
TEST_CASE("copy d2d size 0", "[copy][d2d][zero_size]")
|
||||
{
|
||||
namespace cudax = cuda::experimental;
|
||||
thrust::device_vector<int> d_src(1, 0);
|
||||
thrust::device_vector<int> d_dst(1, 0);
|
||||
|
||||
using extents_t = cuda::std::dextents<int, 2>;
|
||||
extents_t ext(0, 0);
|
||||
cuda::std::layout_right::mapping<extents_t> mapping(ext);
|
||||
|
||||
cuda::device_mdspan<int, extents_t, layout_right> src(thrust::raw_pointer_cast(d_src.data()), mapping);
|
||||
cuda::device_mdspan<int, extents_t, layout_right> dst(thrust::raw_pointer_cast(d_dst.data()), mapping);
|
||||
|
||||
cudax::copy(src, dst, stream);
|
||||
stream.sync();
|
||||
|
||||
thrust::host_vector<int> result(d_dst);
|
||||
REQUIRE(result[0] == 0);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Non-default accessor (scaled_accessor: scales element values by 2 on read)
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (128):(1) with scaled_accessor (scale factor = 2)
|
||||
// dst: (128):(1) with default_accessor
|
||||
// Bypasses the DeviceTransform contiguous path since __have_default_accessors is false
|
||||
TEST_CASE("copy d2d contiguous scaled_accessor", "[copy][d2d][1d][accessor]")
|
||||
{
|
||||
namespace cudax = cuda::experimental;
|
||||
constexpr int N = 128;
|
||||
|
||||
auto h_src = make_iota<int>(N);
|
||||
thrust::device_vector<int> d_src(h_src.begin(), h_src.end());
|
||||
thrust::device_vector<int> d_dst(N, 0);
|
||||
|
||||
using extents_t = cuda::std::dextents<int, 1>;
|
||||
using scaled_acc_t = cuda::std::linalg::scaled_accessor<int, cuda::std::default_accessor<int>>;
|
||||
using dev_acc_t = cuda::device_accessor<scaled_acc_t>;
|
||||
using src_mdspan_t = cuda::device_mdspan<const int, extents_t, layout_right, scaled_acc_t>;
|
||||
using dst_mdspan_t = cuda::device_mdspan<int, extents_t, layout_right>;
|
||||
extents_t ext(N);
|
||||
layout_right::mapping<extents_t> mapping(ext);
|
||||
|
||||
src_mdspan_t src(
|
||||
thrust::raw_pointer_cast(d_src.data()), mapping, dev_acc_t{scaled_acc_t{2, cuda::std::default_accessor<int>{}}});
|
||||
dst_mdspan_t dst(thrust::raw_pointer_cast(d_dst.data()), mapping);
|
||||
|
||||
cudax::copy(src, dst, stream);
|
||||
stream.sync();
|
||||
|
||||
thrust::host_vector<int> h_expected(N);
|
||||
for (int i = 0; i < N; ++i)
|
||||
{
|
||||
h_expected[i] = i * 2;
|
||||
}
|
||||
thrust::host_vector<int> result(d_dst);
|
||||
REQUIRE(result == h_expected);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Large element type (64 bytes)
|
||||
**********************************************************************************************************************/
|
||||
|
||||
struct alignas(64) large_type_64
|
||||
{
|
||||
cuda::std::array<char, 64> data;
|
||||
|
||||
friend bool operator==(const large_type_64& __lhs, const large_type_64& __rhs)
|
||||
{
|
||||
return __lhs.data == __rhs.data;
|
||||
}
|
||||
};
|
||||
|
||||
// src: (128):(1)
|
||||
// dst: (128):(1)
|
||||
TEST_CASE("copy d2d large element 64 bytes", "[copy][d2d][large_element]")
|
||||
{
|
||||
constexpr int N = 128;
|
||||
thrust::host_vector<large_type_64> data(N);
|
||||
for (int i = 0; i < N; ++i)
|
||||
{
|
||||
for (int j = 0; j < 64; ++j)
|
||||
{
|
||||
data[i].data[j] = static_cast<char>((i * 64 + j) % 128);
|
||||
}
|
||||
}
|
||||
test_copy<layout_right>(data, N);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Large element type (128 bytes), non-vectorizable contiguous kernel
|
||||
**********************************************************************************************************************/
|
||||
|
||||
struct alignas(64) large_type_128
|
||||
{
|
||||
cuda::std::array<char, 128> data;
|
||||
|
||||
friend bool operator==(const large_type_128& __lhs, const large_type_128& __rhs)
|
||||
{
|
||||
return __lhs.data == __rhs.data;
|
||||
}
|
||||
};
|
||||
|
||||
// src: (4,512):(512,1), layout_right, sizeof(T) = 128
|
||||
// dst: (4,512):(512,1), layout_right, sizeof(T) = 128
|
||||
// sizeof(T) > __max_vector_access → __are_vectorizable_copy is false
|
||||
// inner_extent_bytes = 512 * 128 = 64KB >= __bytes_in_flight → path (2b)
|
||||
// 2D with outer dim > 1 → tile_size != tensor_size → bypasses DeviceTransform path (1)
|
||||
TEST_CASE("copy d2d large element 128 bytes 2D contiguous", "[copy][d2d][large_element]")
|
||||
{
|
||||
constexpr int M = 4;
|
||||
constexpr int N = 512;
|
||||
thrust::host_vector<large_type_128> data(M * N);
|
||||
for (int i = 0; i < M * N; ++i)
|
||||
{
|
||||
for (int j = 0; j < 128; ++j)
|
||||
{
|
||||
data[i].data[j] = static_cast<char>((i * 128 + j) % 128);
|
||||
}
|
||||
}
|
||||
test_copy<layout_right>(data, M, N);
|
||||
}
|
||||
|
||||
// src: (2,513):(1024,1), layout_stride, sizeof(T) = 128
|
||||
// dst: (2,513):(1024,1), layout_stride
|
||||
// Padded outer stride (1024 > 513) prevents coalescing → tile_size = 513 != tensor_size = 1026
|
||||
// inner_extent_bytes = 513 * 128 = 65664 >= __bytes_in_flight → path (2b)
|
||||
// __are_vectorizable_copy is false (alignof(128) > __max_vector_access)
|
||||
// _TileSize = 256, inner_size = 513 = 2 * 256 + 1 → last tile has 1 remaining element
|
||||
TEST_CASE("copy d2d contiguous kernel remainder", "[copy][d2d][contiguous][remainder]")
|
||||
{
|
||||
constexpr int M = 2;
|
||||
constexpr int N = 513;
|
||||
constexpr int Ld = 1024;
|
||||
|
||||
cuda::std::array<int, 2> shape{M, N};
|
||||
cuda::std::array<int, 2> strides{Ld, 1};
|
||||
|
||||
using extents_t = cuda::std::dextents<int, 2>;
|
||||
using mapping_t = cuda::std::layout_stride::mapping<extents_t>;
|
||||
const int span_size = static_cast<int>(mapping_t(extents_t(shape), strides).required_span_size());
|
||||
|
||||
thrust::host_vector<large_type_128> h_src(span_size);
|
||||
for (int i = 0; i < span_size; ++i)
|
||||
{
|
||||
for (int j = 0; j < 128; ++j)
|
||||
{
|
||||
h_src[i].data[j] = static_cast<char>((i * 128 + j) % 128);
|
||||
}
|
||||
}
|
||||
|
||||
large_type_128 zero{};
|
||||
thrust::host_vector<large_type_128> h_expected(span_size, zero);
|
||||
for (int i = 0; i < M; ++i)
|
||||
{
|
||||
for (int j = 0; j < N; ++j)
|
||||
{
|
||||
h_expected[static_cast<std::size_t>(i) * Ld + j] = h_src[static_cast<std::size_t>(i) * Ld + j];
|
||||
}
|
||||
}
|
||||
|
||||
test_copy_strided(h_src, h_expected, shape, strides, strides);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Overaligned type (alignof > natural alignment, vectorizable via alignment-based check)
|
||||
**********************************************************************************************************************/
|
||||
|
||||
struct alignas(16) overaligned_int
|
||||
{
|
||||
int value;
|
||||
|
||||
bool operator==(const overaligned_int& other) const
|
||||
{
|
||||
return value == other.value;
|
||||
}
|
||||
};
|
||||
|
||||
static_assert(sizeof(overaligned_int) == 16);
|
||||
static_assert(alignof(overaligned_int) == 16);
|
||||
static_assert(alignof(overaligned_int) >= sizeof(overaligned_int));
|
||||
|
||||
// src: (2,4096):(4096,1), layout_right, sizeof(T) = 16, alignof(T) = 16
|
||||
// dst: (2,4096):(4096,1), layout_right
|
||||
// alignof(T) <= __max_vector_access → __are_vectorizable_copy is true
|
||||
// inner_extent_bytes = 4096 * 16 = 64KB >= __bytes_in_flight → path (2a)
|
||||
// 2D with outer dim > 1 → tile_size != tensor_size → bypasses DeviceTransform path (1)
|
||||
TEST_CASE("copy d2d overaligned type vectorized", "[copy][d2d][overaligned]")
|
||||
{
|
||||
constexpr int M = 2;
|
||||
constexpr int N = 4096;
|
||||
constexpr int total = M * N;
|
||||
thrust::host_vector<overaligned_int> data(total);
|
||||
for (int i = 0; i < total; ++i)
|
||||
{
|
||||
data[i].value = i;
|
||||
}
|
||||
test_copy<layout_right>(data, M, N);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Tile Boundary
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (1024):(1)
|
||||
// dst: (1024):(1)
|
||||
TEST_CASE("copy d2d tile boundary exact", "[copy][d2d][tile_boundary]")
|
||||
{
|
||||
constexpr int N = 1024;
|
||||
test_copy<layout_right>(make_iota<int>(N), N);
|
||||
}
|
||||
|
||||
// src: (1020):(1)
|
||||
// dst: (1020):(1)
|
||||
TEST_CASE("copy d2d tile boundary sub-tile", "[copy][d2d][tile_boundary]")
|
||||
{
|
||||
constexpr int N = 1020;
|
||||
test_copy<layout_right>(make_iota<int>(N), N);
|
||||
}
|
||||
|
||||
// src: (1028):(1)
|
||||
// dst: (1028):(1)
|
||||
TEST_CASE("copy d2d tile boundary partial", "[copy][d2d][tile_boundary]")
|
||||
{
|
||||
constexpr int N = 1028;
|
||||
test_copy<layout_right>(make_iota<int>(N), N);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Negative: mismatched shapes
|
||||
**********************************************************************************************************************/
|
||||
|
||||
TEST_CASE("copy d2d mismatched shapes", "[copy][d2d][negative]")
|
||||
{
|
||||
namespace cudax = cuda::experimental;
|
||||
constexpr int N = 64;
|
||||
thrust::device_vector<float> d_src(N);
|
||||
thrust::device_vector<float> d_dst(N);
|
||||
|
||||
using extents_src_t = cuda::std::dextents<int, 2>;
|
||||
using extents_dst_t = cuda::std::dextents<int, 2>;
|
||||
extents_src_t src_ext(8, 8);
|
||||
extents_dst_t dst_ext(4, 16);
|
||||
|
||||
cuda::device_mdspan<float, extents_src_t, layout_right> src(
|
||||
thrust::raw_pointer_cast(d_src.data()), layout_right::mapping<extents_src_t>(src_ext));
|
||||
cuda::device_mdspan<float, extents_dst_t, layout_right> dst(
|
||||
thrust::raw_pointer_cast(d_dst.data()), layout_right::mapping<extents_dst_t>(dst_ext));
|
||||
|
||||
CHECK_THROWS_AS(cudax::copy(src, dst, stream), std::invalid_argument);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Mismatched extents/strides types between src and dst
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: dextents<int, 2>(4, 8), layout_right
|
||||
// dst: dextents<long long, 2>(4, 8), layout_right
|
||||
TEST_CASE("copy d2d different extent types", "[copy][d2d][mixed_types]")
|
||||
{
|
||||
namespace cudax = cuda::experimental;
|
||||
constexpr int M = 4;
|
||||
constexpr int N = 8;
|
||||
|
||||
auto h_data = make_iota<float>(M * N);
|
||||
thrust::device_vector<float> d_src(h_data.begin(), h_data.end());
|
||||
thrust::device_vector<float> d_dst(M * N, 0.0f);
|
||||
|
||||
using src_extents_t = cuda::std::dextents<int, 2>;
|
||||
using dst_extents_t = cuda::std::dextents<long long, 2>;
|
||||
src_extents_t src_ext(M, N);
|
||||
dst_extents_t dst_ext(M, N);
|
||||
|
||||
cuda::device_mdspan<float, src_extents_t, layout_right> src(
|
||||
thrust::raw_pointer_cast(d_src.data()), layout_right::mapping<src_extents_t>(src_ext));
|
||||
cuda::device_mdspan<float, dst_extents_t, layout_right> dst(
|
||||
thrust::raw_pointer_cast(d_dst.data()), layout_right::mapping<dst_extents_t>(dst_ext));
|
||||
|
||||
cudax::copy(src, dst, stream);
|
||||
stream.sync();
|
||||
|
||||
thrust::host_vector<float> result(d_dst);
|
||||
REQUIRE(result == h_data);
|
||||
}
|
||||
|
||||
// src: dextents<int, 2>(4, 8), layout_stride with strides array<int, 2>
|
||||
// dst: dextents<long long, 2>(4, 8), layout_stride with strides array<long long, 2>
|
||||
TEST_CASE("copy d2d different extent and stride types", "[copy][d2d][mixed_types]")
|
||||
{
|
||||
namespace cudax = cuda::experimental;
|
||||
constexpr int M = 4;
|
||||
constexpr int N = 8;
|
||||
|
||||
auto h_data = make_iota<float>(M * N);
|
||||
thrust::device_vector<float> d_src(h_data.begin(), h_data.end());
|
||||
thrust::device_vector<float> d_dst(M * N, 0.0f);
|
||||
|
||||
using src_extents_t = cuda::std::dextents<int, 2>;
|
||||
using dst_extents_t = cuda::std::dextents<long long, 2>;
|
||||
using src_mapping_t = cuda::std::layout_stride::mapping<src_extents_t>;
|
||||
using dst_mapping_t = cuda::std::layout_stride::mapping<dst_extents_t>;
|
||||
|
||||
src_mapping_t src_mapping(src_extents_t(M, N), cuda::std::array<int, 2>{N, 1});
|
||||
dst_mapping_t dst_mapping(dst_extents_t(M, N), cuda::std::array<long long, 2>{N, 1});
|
||||
|
||||
cuda::device_mdspan<float, src_extents_t, cuda::std::layout_stride> src(
|
||||
thrust::raw_pointer_cast(d_src.data()), src_mapping);
|
||||
cuda::device_mdspan<float, dst_extents_t, cuda::std::layout_stride> dst(
|
||||
thrust::raw_pointer_cast(d_dst.data()), dst_mapping);
|
||||
|
||||
cudax::copy(src, dst, stream);
|
||||
stream.sync();
|
||||
|
||||
thrust::host_vector<float> result(d_dst);
|
||||
REQUIRE(result == h_data);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Misaligned pointer + vectorization
|
||||
**********************************************************************************************************************/
|
||||
|
||||
TEST_CASE("copy d2d misaligned pointer", "[copy][d2d][alignment]")
|
||||
{
|
||||
namespace cudax = cuda::experimental;
|
||||
constexpr int N = 512;
|
||||
|
||||
thrust::host_vector<char> h_src(N);
|
||||
for (int i = 0; i < N; ++i)
|
||||
{
|
||||
h_src[i] = static_cast<char>(i % 128);
|
||||
}
|
||||
|
||||
thrust::host_vector<char> h_src_padded(N + 1, char{0});
|
||||
thrust::copy(h_src.begin(), h_src.end(), h_src_padded.begin() + 1);
|
||||
|
||||
thrust::device_vector<char> d_src_buf = h_src_padded;
|
||||
thrust::device_vector<char> d_dst_buf(N + 1, char{0});
|
||||
|
||||
auto* src_ptr = thrust::raw_pointer_cast(d_src_buf.data()) + 1;
|
||||
auto* dst_ptr = thrust::raw_pointer_cast(d_dst_buf.data()) + 1;
|
||||
|
||||
using extents_t = cuda::std::dextents<int, 1>;
|
||||
extents_t ext(N);
|
||||
layout_right::mapping<extents_t> mapping(ext);
|
||||
|
||||
cuda::device_mdspan<char, extents_t, layout_right> src(src_ptr, mapping);
|
||||
cuda::device_mdspan<char, extents_t, layout_right> dst(dst_ptr, mapping);
|
||||
|
||||
cudax::copy(src, dst, stream);
|
||||
stream.sync();
|
||||
|
||||
thrust::host_vector<char> h_dst_buf(d_dst_buf);
|
||||
thrust::host_vector<char> result(h_dst_buf.begin() + 1, h_dst_buf.begin() + 1 + N);
|
||||
REQUIRE(result == h_src);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Large count > INT_MAX
|
||||
**********************************************************************************************************************/
|
||||
|
||||
TEST_CASE("copy d2d large count > INT_MAX", "[copy][d2d][large][.]")
|
||||
{
|
||||
namespace cudax = cuda::experimental;
|
||||
|
||||
const auto N = static_cast<size_t>(INT_MAX) + 257;
|
||||
const auto required = 2 * N;
|
||||
|
||||
size_t free_mem = 0;
|
||||
size_t total_mem = 0;
|
||||
cudaMemGetInfo(&free_mem, &total_mem);
|
||||
if (free_mem < required + (size_t{256} << 20))
|
||||
{
|
||||
SKIP("Not enough GPU memory (" << (free_mem >> 20) << " MB free, need ~" << (required >> 20) << " MB)");
|
||||
}
|
||||
|
||||
thrust::device_vector<char> d_src(N, static_cast<char>(0x42));
|
||||
thrust::device_vector<char> d_dst(N, static_cast<char>(0x00));
|
||||
|
||||
using extents_t = cuda::std::dextents<long long, 1>;
|
||||
extents_t ext(static_cast<long long>(N));
|
||||
layout_right::mapping<extents_t> mapping(ext);
|
||||
|
||||
cuda::device_mdspan<char, extents_t, layout_right> src(thrust::raw_pointer_cast(d_src.data()), mapping);
|
||||
cuda::device_mdspan<char, extents_t, layout_right> dst(thrust::raw_pointer_cast(d_dst.data()), mapping);
|
||||
|
||||
cudax::copy(src, dst, stream);
|
||||
stream.sync();
|
||||
|
||||
REQUIRE(d_dst[0] == static_cast<char>(0x42));
|
||||
REQUIRE(d_dst[N / 2] == static_cast<char>(0x42));
|
||||
REQUIRE(d_dst[N - 1] == static_cast<char>(0x42));
|
||||
}
|
||||
@@ -1,228 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include "copy_common.cuh"
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* LLM Prefill GPU Tensors (large batch)
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (19456,2880):(2880,1), BF16, ~107 MB
|
||||
// dst: (19456,2880):(2880,1)
|
||||
TEST_CASE("copy d2d llm prefill hidden_states", "[copy][d2d][llm][prefill]")
|
||||
{
|
||||
test_copy_iota<int16_t>(19456, 2880);
|
||||
}
|
||||
|
||||
// src: (19456,2560):(2560,1), BF16, ~95 MB
|
||||
// dst: (19456,2560):(2560,1)
|
||||
TEST_CASE("copy d2d llm prefill qkv_fused", "[copy][d2d][llm][prefill]")
|
||||
{
|
||||
test_copy_iota<int16_t>(19456, 2560);
|
||||
}
|
||||
|
||||
// src: (19456,2048):(2048,1), BF16, ~76 MB
|
||||
// dst: (19456,2048):(2048,1)
|
||||
TEST_CASE("copy d2d llm prefill q_output", "[copy][d2d][llm][prefill]")
|
||||
{
|
||||
test_copy_iota<int16_t>(19456, 2048);
|
||||
}
|
||||
|
||||
// src: (19456,128):(128,1), BF16, ~4.75 MB
|
||||
// dst: (19456,128):(128,1)
|
||||
TEST_CASE("copy d2d llm prefill router_logits", "[copy][d2d][llm][prefill]")
|
||||
{
|
||||
test_copy_iota<int16_t>(19456, 128);
|
||||
}
|
||||
|
||||
// src: (19456,4):(4,1), int32, ~304 KB
|
||||
// dst: (19456,4):(4,1)
|
||||
TEST_CASE("copy d2d llm prefill expert_indices", "[copy][d2d][llm][prefill]")
|
||||
{
|
||||
test_copy_iota<int32_t>(19456, 4);
|
||||
}
|
||||
|
||||
// src: (19456,4):(4,1), float32, ~304 KB
|
||||
// dst: (19456,4):(4,1)
|
||||
TEST_CASE("copy d2d llm prefill expert_weights", "[copy][d2d][llm][prefill]")
|
||||
{
|
||||
test_copy_iota<float>(19456, 4);
|
||||
}
|
||||
|
||||
// src: (156608):(1), int32, ~612 KB
|
||||
// dst: (156608):(1)
|
||||
TEST_CASE("copy d2d llm prefill expanded_indices_int32", "[copy][d2d][llm][prefill]")
|
||||
{
|
||||
test_copy_iota<int32_t>(156608);
|
||||
}
|
||||
|
||||
// src: (156608):(1), int64, ~1.2 MB
|
||||
// dst: (156608):(1)
|
||||
TEST_CASE("copy d2d llm prefill expanded_indices_int64", "[copy][d2d][llm][prefill]")
|
||||
{
|
||||
test_copy_iota<int64_t>(156608);
|
||||
}
|
||||
|
||||
// src: (39152,2880):(2880,1), BF16, ~215 MB
|
||||
// dst: (39152,2880):(2880,1)
|
||||
TEST_CASE("copy d2d llm prefill moe_hidden_states", "[copy][d2d][llm][prefill]")
|
||||
{
|
||||
test_copy_iota<int16_t>(39152, 2880);
|
||||
}
|
||||
|
||||
// src: (39152,2880):(2880,1), FP8 E4M3, ~107.5 MB
|
||||
// dst: (39152,2880):(2880,1)
|
||||
TEST_CASE("copy d2d llm prefill quantized_activations", "[copy][d2d][llm][prefill]")
|
||||
{
|
||||
test_copy_iota<int8_t>(39152, 2880);
|
||||
}
|
||||
|
||||
// src: (20,201088):(201088,1), float32, ~15.3 MB
|
||||
// dst: (20,201088):(201088,1)
|
||||
TEST_CASE("copy d2d llm prefill logits", "[copy][d2d][llm][prefill]")
|
||||
{
|
||||
test_copy_iota<float>(20, 201088);
|
||||
}
|
||||
|
||||
// src: (20):(1), int32, 80 B
|
||||
// dst: (20):(1)
|
||||
TEST_CASE("copy d2d llm prefill sampled_tokens_int32", "[copy][d2d][llm][prefill]")
|
||||
{
|
||||
test_copy_iota<int32_t>(20);
|
||||
}
|
||||
|
||||
// src: (20):(1), int64, 160 B
|
||||
// dst: (20):(1)
|
||||
TEST_CASE("copy d2d llm prefill sampled_tokens_int64", "[copy][d2d][llm][prefill]")
|
||||
{
|
||||
test_copy_iota<int64_t>(20);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* LLM Decode Tensors (small batch)
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (1792,2880):(2880,1), BF16, ~9.8 MB
|
||||
// dst: (1792,2880):(2880,1)
|
||||
TEST_CASE("copy d2d llm decode hidden_states", "[copy][d2d][llm][decode]")
|
||||
{
|
||||
test_copy_iota<int16_t>(1792, 2880);
|
||||
}
|
||||
|
||||
// src: (1792,2560):(2560,1), BF16, ~8.75 MB
|
||||
// dst: (1792,2560):(2560,1)
|
||||
TEST_CASE("copy d2d llm decode qkv_fused", "[copy][d2d][llm][decode]")
|
||||
{
|
||||
test_copy_iota<int16_t>(1792, 2560);
|
||||
}
|
||||
|
||||
// src: (1792,2048):(2048,1), BF16, 7 MB
|
||||
// dst: (1792,2048):(2048,1)
|
||||
TEST_CASE("copy d2d llm decode q_output", "[copy][d2d][llm][decode]")
|
||||
{
|
||||
test_copy_iota<int16_t>(1792, 2048);
|
||||
}
|
||||
|
||||
// src: (1792,128):(128,1), BF16, 448 KB
|
||||
// dst: (1792,128):(128,1)
|
||||
TEST_CASE("copy d2d llm decode router_logits", "[copy][d2d][llm][decode]")
|
||||
{
|
||||
test_copy_iota<int16_t>(1792, 128);
|
||||
}
|
||||
|
||||
// src: (1792,4):(4,1), int32, 28 KB
|
||||
// dst: (1792,4):(4,1)
|
||||
TEST_CASE("copy d2d llm decode expert_indices", "[copy][d2d][llm][decode]")
|
||||
{
|
||||
test_copy_iota<int32_t>(1792, 4);
|
||||
}
|
||||
|
||||
// src: (1792,4):(4,1), float32, 28 KB
|
||||
// dst: (1792,4):(4,1)
|
||||
TEST_CASE("copy d2d llm decode expert_weights", "[copy][d2d][llm][decode]")
|
||||
{
|
||||
test_copy_iota<float>(1792, 4);
|
||||
}
|
||||
|
||||
// src: (14336):(1), int32, 56 KB
|
||||
// dst: (14336):(1)
|
||||
TEST_CASE("copy d2d llm decode expanded_indices_int32", "[copy][d2d][llm][decode]")
|
||||
{
|
||||
test_copy_iota<int32_t>(14336);
|
||||
}
|
||||
|
||||
// src: (14336):(1), int64, 112 KB
|
||||
// dst: (14336):(1)
|
||||
TEST_CASE("copy d2d llm decode expanded_indices_int64", "[copy][d2d][llm][decode]")
|
||||
{
|
||||
test_copy_iota<int64_t>(14336);
|
||||
}
|
||||
|
||||
// src: (3584,2880):(2880,1), BF16, ~19.7 MB
|
||||
// dst: (3584,2880):(2880,1)
|
||||
TEST_CASE("copy d2d llm decode moe_hidden_states", "[copy][d2d][llm][decode]")
|
||||
{
|
||||
test_copy_iota<int16_t>(3584, 2880);
|
||||
}
|
||||
|
||||
// src: (3584,2880):(2880,1), FP8 E4M3, ~9.8 MB
|
||||
// dst: (3584,2880):(2880,1)
|
||||
TEST_CASE("copy d2d llm decode quantized_activations", "[copy][d2d][llm][decode]")
|
||||
{
|
||||
test_copy_iota<int8_t>(3584, 2880);
|
||||
}
|
||||
|
||||
// src: (1792,201088):(201088,1), float32, ~1.34 GB
|
||||
// dst: (1792,201088):(201088,1)
|
||||
TEST_CASE("copy d2d llm decode logits", "[copy][d2d][llm][decode]")
|
||||
{
|
||||
test_copy_iota<float>(1792, 201088);
|
||||
}
|
||||
|
||||
// src: (1792):(1), int32, 7 KB
|
||||
// dst: (1792):(1)
|
||||
TEST_CASE("copy d2d llm decode sampled_tokens_int32", "[copy][d2d][llm][decode]")
|
||||
{
|
||||
test_copy_iota<int32_t>(1792);
|
||||
}
|
||||
|
||||
// src: (1792):(1), int64, 14 KB
|
||||
// dst: (1792):(1)
|
||||
TEST_CASE("copy d2d llm decode sampled_tokens_int64", "[copy][d2d][llm][decode]")
|
||||
{
|
||||
test_copy_iota<int64_t>(1792);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* LLM Static / Weight Tensors
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (2880):(1), BF16, ~5.6 KB
|
||||
// dst: (2880):(1)
|
||||
TEST_CASE("copy d2d llm weight rmsnorm", "[copy][d2d][llm][weight]")
|
||||
{
|
||||
test_copy_iota<int16_t>(2880);
|
||||
}
|
||||
|
||||
// src: (2560):(1), BF16, 5 KB
|
||||
// dst: (2560):(1)
|
||||
TEST_CASE("copy d2d llm bias qkv", "[copy][d2d][llm][weight]")
|
||||
{
|
||||
test_copy_iota<int16_t>(2560);
|
||||
}
|
||||
|
||||
// src: (131072,32):(32,1), float2 (8 bytes, using int64_t as surrogate), 32 MB
|
||||
// dst: (131072,32):(32,1)
|
||||
TEST_CASE("copy d2d llm rope_cache", "[copy][d2d][llm][weight]")
|
||||
{
|
||||
test_copy_iota<int64_t>(131072, 32);
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include "copy_common.cuh"
|
||||
|
||||
using data_t = int8_t;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* nvmath minimal offset test (device-to-device)
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (3,4):(4,1), offset=4, alloc=16
|
||||
// dst: (3,4):(4,1), offset=0, alloc=12
|
||||
TEST_CASE("copy d2d nvmath minimal offset", "[copy][d2d][nvmath][debug]")
|
||||
{
|
||||
cuda::std::array<int, 2> shape{3, 4};
|
||||
cuda::std::array<int, 2> src_strides{4, 1};
|
||||
cuda::std::array<int, 2> dst_strides{4, 1};
|
||||
test_copy_stride_relaxed<data_t>(16, 4, shape, src_strides, 12, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (4,256):(512,1), offset=256, alloc=2048+256
|
||||
// dst: (4,256):(256,1), offset=0, alloc=1024
|
||||
TEST_CASE("copy d2d nvmath medium offset", "[copy][d2d][nvmath][debug]")
|
||||
{
|
||||
cuda::std::array<int, 2> shape{4, 256};
|
||||
cuda::std::array<int, 2> src_strides{512, 1};
|
||||
cuda::std::array<int, 2> dst_strides{256, 1};
|
||||
test_copy_stride_relaxed<data_t>(2048 + 256, 256, shape, src_strides, 1024, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (35,255,10,24):(61440,240,24,1), offset=240, alloc=35*256*10*24 — same as sliced_vec but smaller
|
||||
// dst: (3,255,10,24):(61200,240,24,1), offset=0, alloc=3*255*10*24
|
||||
TEST_CASE("copy d2d nvmath small sliced_vec", "[copy][d2d][nvmath][debug]")
|
||||
{
|
||||
constexpr int src_alloc = 3 * 256 * 10 * 24;
|
||||
constexpr int dst_alloc = 3 * 255 * 10 * 24;
|
||||
constexpr int src_offset = 240;
|
||||
cuda::std::array<int, 4> shape{3, 255, 10, 24};
|
||||
cuda::std::array<int, 4> src_strides{61440, 240, 24, 1};
|
||||
cuda::std::array<int, 4> dst_strides{61200, 240, 24, 1};
|
||||
test_copy_stride_relaxed<data_t>(src_alloc, src_offset, shape, src_strides, dst_alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* nvmath memcpy test cases (device-to-device)
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (70,90,80,80):(576000,6400,80,1)
|
||||
// dst: (70,90,80,80):(576000,6400,80,1)
|
||||
TEST_CASE("copy d2d nvmath memcpy_layout_0", "[copy][d2d][nvmath][memcpy]")
|
||||
{
|
||||
test_copy_iota<data_t>(70, 90, 80, 80);
|
||||
}
|
||||
|
||||
// src: (70,90,80,80):(90,1,6300,504000)
|
||||
// dst: (70,90,80,80):(90,1,6300,504000)
|
||||
TEST_CASE("copy d2d nvmath memcpy_layout_1", "[copy][d2d][nvmath][memcpy]")
|
||||
{
|
||||
constexpr int alloc = 70 * 90 * 80 * 80;
|
||||
test_copy_strided(
|
||||
make_iota<data_t>(alloc), cuda::std::array<int, 4>{70, 90, 80, 80}, cuda::std::array<int, 4>{90, 1, 6300, 504000});
|
||||
}
|
||||
|
||||
// src: (1001,1007,3,31):(31217,1,31248217,1007), offset=0, alloc=1001*1007*3*31
|
||||
// dst: (1001,1007,3,31):(31217,1,31248217,1007), offset=31248217, alloc=1001*1007*5*31
|
||||
TEST_CASE("copy d2d nvmath memcpy_layout_2", "[copy][d2d][nvmath][memcpy]")
|
||||
{
|
||||
constexpr int src_alloc = 1001 * 1007 * 3 * 31;
|
||||
constexpr int dst_alloc = 1001 * 1007 * 5 * 31;
|
||||
constexpr int dst_offset = 31248217;
|
||||
cuda::std::array<int, 4> shape{1001, 1007, 3, 31};
|
||||
cuda::std::array<int, 4> strides{31217, 1, 31248217, 1007};
|
||||
test_copy_stride_relaxed<data_t>(src_alloc, 0, shape, strides, dst_alloc, dst_offset, strides);
|
||||
}
|
||||
|
||||
// src: (57,71,1,1007,1):(71497,1,12225987,71,4075329), offset=12225987+4075329, alloc=57*71*3*1007*3
|
||||
// dst: (57,71,1,1007,1):(71497,1,4075329,71,20376645), offset=3*4075329, alloc=57*71*5*1007*1
|
||||
TEST_CASE("copy d2d nvmath memcpy_layout_3", "[copy][d2d][nvmath][memcpy]")
|
||||
{
|
||||
constexpr int src_alloc = 57 * 71 * 3 * 1007 * 3;
|
||||
constexpr int src_offset = 12225987 + 4075329;
|
||||
constexpr int dst_alloc = 57 * 71 * 5 * 1007 * 1;
|
||||
constexpr int dst_offset = 3 * 4075329;
|
||||
cuda::std::array<int, 5> shape{57, 71, 1, 1007, 1};
|
||||
cuda::std::array<int, 5> src_strides{71497, 1, 12225987, 71, 4075329};
|
||||
cuda::std::array<int, 5> dst_strides{71497, 1, 4075329, 71, 20376645};
|
||||
test_copy_stride_relaxed<data_t>(src_alloc, src_offset, shape, src_strides, dst_alloc, dst_offset, dst_strides);
|
||||
}
|
||||
|
||||
// src: (63,70,1001):(1001,63063,-1), offset=1000
|
||||
// dst: (63,70,1001):(1001,63063,-1), offset=1000
|
||||
TEST_CASE("copy d2d nvmath memcpy_neg", "[copy][d2d][nvmath][memcpy]")
|
||||
{
|
||||
constexpr int alloc = 63 * 70 * 1001;
|
||||
constexpr int offset = 1000;
|
||||
cuda::std::array<int, 3> shape{63, 70, 1001};
|
||||
cuda::std::array<int, 3> strides{1001, 63063, -1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, offset, shape, strides);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* nvmath reorder strides test case (device-to-device)
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (8,100019,4):(1100209,11,3), offset=0, alloc=8*100019*11
|
||||
// dst: (8,100019,4):(1,8,800152), offset=0, alloc=8*100019*4
|
||||
TEST_CASE("copy d2d nvmath reorder_strides", "[copy][d2d][nvmath][reorder_strides]")
|
||||
{
|
||||
constexpr int src_alloc = 8 * 100019 * 11;
|
||||
constexpr int dst_alloc = 8 * 100019 * 4;
|
||||
cuda::std::array<int, 3> shape{8, 100019, 4};
|
||||
cuda::std::array<int, 3> src_strides{1100209, 11, 3};
|
||||
cuda::std::array<int, 3> dst_strides{1, 8, 800152};
|
||||
test_copy_stride_relaxed<data_t>(src_alloc, 0, shape, src_strides, dst_alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* nvmath negative strides test case (device-to-device)
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (63,70,1001):(-1001,-63063,-1), offset=alloc-1
|
||||
// dst: (63,70,1001):(70070,1001,1), offset=0
|
||||
TEST_CASE("copy d2d nvmath src_neg_stride", "[copy][d2d][nvmath][neg_stride]")
|
||||
{
|
||||
constexpr int alloc = 63 * 70 * 1001;
|
||||
constexpr int src_offset = alloc - 1;
|
||||
cuda::std::array<int, 3> shape{63, 70, 1001};
|
||||
cuda::std::array<int, 3> src_strides{-1001, -63063, -1};
|
||||
cuda::std::array<int, 3> dst_strides{70070, 1001, 1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, src_offset, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* nvmath squeezing and flattening test cases (device-to-device)
|
||||
**********************************************************************************************************************/
|
||||
|
||||
static cuda::std::array<int, 23> make_flatten_common_src_strides()
|
||||
{
|
||||
return {1 << 15, 1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20, 1 << 21, 1 << 22, 1 << 14, 1 << 13, 1 << 12, 1 << 11,
|
||||
1 << 10, 1 << 9, 1 << 8, 1 << 7, 1 << 6, 1 << 5, 1 << 4, 1 << 3, 1 << 2, 1 << 0, 1 << 1};
|
||||
}
|
||||
|
||||
static cuda::std::array<int, 23> make_flatten_common_dst_strides()
|
||||
{
|
||||
return {1 << 15, 1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20, 1 << 21, 1 << 22, 1 << 14, 1 << 13, 1 << 12, 1 << 11,
|
||||
1 << 10, 1 << 9, 1 << 8, 1 << 7, 1 << 6, 1 << 5, 1 << 4, 1 << 3, 1 << 1, 1 << 2, 1 << 0};
|
||||
}
|
||||
|
||||
// src: (2,)^23, bit-permuted strides
|
||||
// dst: (2,)^23, different bit-permuted strides
|
||||
TEST_CASE("copy d2d nvmath flatten_common", "[copy][d2d][nvmath][flatten]")
|
||||
{
|
||||
constexpr int alloc = 1 << 23;
|
||||
cuda::std::array<int, 23> shape{};
|
||||
for (auto& s : shape)
|
||||
{
|
||||
s = 2;
|
||||
}
|
||||
test_copy_stride_relaxed<data_t>(
|
||||
alloc, 0, shape, make_flatten_common_src_strides(), alloc, 0, make_flatten_common_dst_strides());
|
||||
}
|
||||
|
||||
// src: (4,2,...,2):(5,2^4,...,2^22), alloc=2^23
|
||||
// dst: (4,2,...,2):(2^19,2^18,...,1), alloc=2^21
|
||||
TEST_CASE("copy d2d nvmath flatten_one", "[copy][d2d][nvmath][flatten]")
|
||||
{
|
||||
constexpr int src_alloc = 1 << 23;
|
||||
constexpr int dst_alloc = 1 << 21;
|
||||
cuda::std::array<int, 20> shape{4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2};
|
||||
cuda::std::array<int, 20> src_strides{
|
||||
5, 1 << 4, 1 << 5, 1 << 6, 1 << 7, 1 << 8, 1 << 9, 1 << 10, 1 << 11, 1 << 12,
|
||||
1 << 13, 1 << 14, 1 << 15, 1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20, 1 << 21, 1 << 22};
|
||||
cuda::std::array<int, 20> dst_strides{
|
||||
1 << 19, 1 << 18, 1 << 17, 1 << 16, 1 << 15, 1 << 14, 1 << 13, 1 << 12, 1 << 11, 1 << 10,
|
||||
1 << 9, 1 << 8, 1 << 7, 1 << 6, 1 << 5, 1 << 4, 1 << 3, 1 << 2, 1 << 1, 1 << 0};
|
||||
test_copy_stride_relaxed<data_t>(src_alloc, 0, shape, src_strides, dst_alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* nvmath vectorize test cases (device-to-device)
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (35,255,10,24):(61440,240,24,1), offset=240, alloc=35*256*10*24
|
||||
// dst: (35,255,10,24):(61200,240,24,1), offset=0, alloc=35*255*10*24
|
||||
TEST_CASE("copy d2d nvmath sliced_vec", "[copy][d2d][nvmath][vectorize]")
|
||||
{
|
||||
constexpr int src_alloc = 35 * 256 * 10 * 24;
|
||||
constexpr int dst_alloc = 35 * 255 * 10 * 24;
|
||||
constexpr int src_offset = 240;
|
||||
cuda::std::array<int, 4> shape{35, 255, 10, 24};
|
||||
cuda::std::array<int, 4> src_strides{61440, 240, 24, 1};
|
||||
cuda::std::array<int, 4> dst_strides{61200, 240, 24, 1};
|
||||
test_copy_stride_relaxed<data_t>(src_alloc, src_offset, shape, src_strides, dst_alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (355,255,4,3):(3072,12,3,1), offset=12, alloc=355*256*4*3
|
||||
// dst: (355,255,4,3):(3060,12,3,1), offset=0, alloc=355*255*4*3
|
||||
TEST_CASE("copy d2d nvmath sliced_vec_2", "[copy][d2d][nvmath][vectorize]")
|
||||
{
|
||||
constexpr int src_alloc = 355 * 256 * 4 * 3;
|
||||
constexpr int dst_alloc = 355 * 255 * 4 * 3;
|
||||
constexpr int src_offset = 12;
|
||||
cuda::std::array<int, 4> shape{355, 255, 4, 3};
|
||||
cuda::std::array<int, 4> src_strides{3072, 12, 3, 1};
|
||||
cuda::std::array<int, 4> dst_strides{3060, 12, 3, 1};
|
||||
test_copy_stride_relaxed<data_t>(src_alloc, src_offset, shape, src_strides, dst_alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (35,255,5,10):(153000,600,20,1), offset=10*20+5=205, alloc=35*255*30*20
|
||||
// dst: (35,255,5,10):(12750,50,10,1), offset=0, alloc=35*255*5*10
|
||||
TEST_CASE("copy d2d nvmath sliced_unaligned_ptr", "[copy][d2d][nvmath][vectorize]")
|
||||
{
|
||||
constexpr int src_alloc = 35 * 255 * 30 * 20;
|
||||
constexpr int dst_alloc = 35 * 255 * 5 * 10;
|
||||
constexpr int src_offset = 10 * 20 + 5;
|
||||
cuda::std::array<int, 4> shape{35, 255, 5, 10};
|
||||
cuda::std::array<int, 4> src_strides{153000, 600, 20, 1};
|
||||
cuda::std::array<int, 4> dst_strides{12750, 50, 10, 1};
|
||||
test_copy_stride_relaxed<data_t>(src_alloc, src_offset, shape, src_strides, dst_alloc, 0, dst_strides);
|
||||
}
|
||||
@@ -1,174 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
// GCC -Warray-bounds false positive for high-rank (20+) __raw_tensor instantiations
|
||||
#if _CCCL_COMPILER(GCC)
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Warray-bounds")
|
||||
#endif // _CCCL_COMPILER(GCC)
|
||||
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include "copy_common.cuh"
|
||||
|
||||
using data_t = int8_t;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* nvmath transpose test cases (device-to-device)
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (2,)^20:(2^19,2^18,...,2^0), C-order
|
||||
// dst: (2,)^20:(2^0,2^1,...,2^19), F-order
|
||||
TEST_CASE("copy d2d nvmath transpose_merge_extents", "[copy][d2d][nvmath][transpose]")
|
||||
{
|
||||
constexpr int alloc = 1 << 20;
|
||||
cuda::std::array<int, 20> shape{};
|
||||
for (auto& s : shape)
|
||||
{
|
||||
s = 2;
|
||||
}
|
||||
// clang-format off
|
||||
cuda::std::array<int, 20> src_strides{
|
||||
1 << 19, 1 << 18, 1 << 17, 1 << 16, 1 << 15, 1 << 14, 1 << 13, 1 << 12,
|
||||
1 << 11, 1 << 10, 1 << 9, 1 << 8, 1 << 7, 1 << 6, 1 << 5, 1 << 4,
|
||||
1 << 3, 1 << 2, 1 << 1, 1 << 0};
|
||||
cuda::std::array<int, 20> dst_strides{
|
||||
1 << 0, 1 << 1, 1 << 2, 1 << 3, 1 << 4, 1 << 5, 1 << 6, 1 << 7,
|
||||
1 << 8, 1 << 9, 1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15,
|
||||
1 << 16, 1 << 17, 1 << 18, 1 << 19};
|
||||
// clang-format on
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (55,13,55,55):(13,1,39325,715)
|
||||
// dst: (55,13,55,55):(39325,3025,55,1)
|
||||
TEST_CASE("copy d2d nvmath transpose", "[copy][d2d][nvmath][transpose]")
|
||||
{
|
||||
constexpr int alloc = 55 * 13 * 55 * 55;
|
||||
cuda::std::array<int, 4> shape{55, 13, 55, 55};
|
||||
cuda::std::array<int, 4> src_strides{13, 1, 39325, 715};
|
||||
cuda::std::array<int, 4> dst_strides{39325, 3025, 55, 1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (55,55,13,55):(715,39325,1,13)
|
||||
// dst: (55,55,13,55):(39325,715,55,1)
|
||||
TEST_CASE("copy d2d nvmath transpose_2", "[copy][d2d][nvmath][transpose]")
|
||||
{
|
||||
constexpr int alloc = 55 * 55 * 13 * 55;
|
||||
cuda::std::array<int, 4> shape{55, 55, 13, 55};
|
||||
cuda::std::array<int, 4> src_strides{715, 39325, 1, 13};
|
||||
cuda::std::array<int, 4> dst_strides{39325, 715, 55, 1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (101,101,101,101):(101,1030301,1,10201)
|
||||
// dst: (101,101,101,101):(1030301,10201,101,1)
|
||||
TEST_CASE("copy d2d nvmath transpose_3", "[copy][d2d][nvmath][transpose]")
|
||||
{
|
||||
constexpr int alloc = 101 * 101 * 101 * 101;
|
||||
cuda::std::array<int, 4> shape{101, 101, 101, 101};
|
||||
cuda::std::array<int, 4> src_strides{101, 1030301, 1, 10201};
|
||||
cuda::std::array<int, 4> dst_strides{1030301, 10201, 101, 1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (7,100019,7):(1,7,700133)
|
||||
// dst: (7,100019,7):(700133,7,1)
|
||||
TEST_CASE("copy d2d nvmath transpose_small_tile", "[copy][d2d][nvmath][transpose]")
|
||||
{
|
||||
constexpr int alloc = 7 * 100019 * 7;
|
||||
cuda::std::array<int, 3> shape{7, 100019, 7};
|
||||
cuda::std::array<int, 3> src_strides{1, 7, 700133};
|
||||
cuda::std::array<int, 3> dst_strides{700133, 7, 1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (33,100019,33):(1,33,3300627)
|
||||
// dst: (33,100019,33):(3300627,33,1)
|
||||
TEST_CASE("copy d2d nvmath transpose_small_tile_2", "[copy][d2d][nvmath][transpose]")
|
||||
{
|
||||
constexpr int alloc = 33 * 100019 * 33;
|
||||
cuda::std::array<int, 3> shape{33, 100019, 33};
|
||||
cuda::std::array<int, 3> src_strides{1, 33, 3300627};
|
||||
cuda::std::array<int, 3> dst_strides{3300627, 33, 1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (7,100019,7):(1,49,7)
|
||||
// dst: (7,100019,7):(700133,1,100019)
|
||||
TEST_CASE("copy d2d nvmath transpose_small_tile_3", "[copy][d2d][nvmath][transpose]")
|
||||
{
|
||||
constexpr int alloc = 7 * 100019 * 7;
|
||||
cuda::std::array<int, 3> shape{7, 100019, 7};
|
||||
cuda::std::array<int, 3> src_strides{1, 49, 7};
|
||||
cuda::std::array<int, 3> dst_strides{700133, 1, 100019};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (33,100019,33):(1,1089,33)
|
||||
// dst: (33,100019,33):(3300627,1,100019)
|
||||
TEST_CASE("copy d2d nvmath transpose_small_tile_4", "[copy][d2d][nvmath][transpose]")
|
||||
{
|
||||
constexpr int alloc = 33 * 100019 * 33;
|
||||
cuda::std::array<int, 3> shape{33, 100019, 33};
|
||||
cuda::std::array<int, 3> src_strides{1, 1089, 33};
|
||||
cuda::std::array<int, 3> dst_strides{3300627, 1, 100019};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (10001,10001,3):(3,30003,1)
|
||||
// dst: (10001,10001,3):(30003,3,1)
|
||||
TEST_CASE("copy d2d nvmath transpose_channels", "[copy][d2d][nvmath][transpose]")
|
||||
{
|
||||
constexpr int alloc = 10001 * 10001 * 3;
|
||||
cuda::std::array<int, 3> shape{10001, 10001, 3};
|
||||
cuda::std::array<int, 3> src_strides{3, 30003, 1};
|
||||
cuda::std::array<int, 3> dst_strides{30003, 3, 1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (1001,1001,3,3,3):(27,27027,9,1,3)
|
||||
// dst: (1001,1001,3,3,3):(27027,27,9,3,1)
|
||||
TEST_CASE("copy d2d nvmath transpose_channels_2", "[copy][d2d][nvmath][transpose]")
|
||||
{
|
||||
constexpr int alloc = 1001 * 1001 * 3 * 3 * 3;
|
||||
cuda::std::array<int, 5> shape{1001, 1001, 3, 3, 3};
|
||||
cuda::std::array<int, 5> src_strides{27, 27027, 9, 1, 3};
|
||||
cuda::std::array<int, 5> dst_strides{27027, 27, 9, 3, 1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (3,1000033):(1,3)
|
||||
// dst: (3,1000033):(1000033,1)
|
||||
TEST_CASE("copy d2d nvmath transpose_inbalanced", "[copy][d2d][nvmath][transpose]")
|
||||
{
|
||||
constexpr int alloc = 3 * 1000033;
|
||||
cuda::std::array<int, 2> shape{3, 1000033};
|
||||
cuda::std::array<int, 2> src_strides{1, 3};
|
||||
cuda::std::array<int, 2> dst_strides{1000033, 1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (4,4,4,4,4,4,16,8):(1,4,...,65536), column-major
|
||||
// dst: (4,4,4,4,4,4,16,8):(131072,...,8,1), row-major
|
||||
// Rank 8 == __max_shared_mem_kernel_rank, verifying the shared-memory kernel is still instantiated at the maximum
|
||||
// allowed rank. The first 6 dimensions fit in one tile (4^6 = 4096 elements); the last 2 dimensions (16x8 = 128 tiles)
|
||||
// provide sufficient grid utilization.
|
||||
TEST_CASE("copy d2d nvmath transpose_max_shared_mem_rank", "[copy][d2d][nvmath][transpose]")
|
||||
{
|
||||
constexpr int alloc = 4 * 4 * 4 * 4 * 4 * 4 * 16 * 8; // 524288
|
||||
cuda::std::array<int, 8> shape{4, 4, 4, 4, 4, 4, 16, 8};
|
||||
cuda::std::array<int, 8> src_strides{1, 4, 16, 64, 256, 1024, 4096, 65536};
|
||||
cuda::std::array<int, 8> dst_strides{131072, 32768, 8192, 2048, 512, 128, 8, 1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "copy_common.cuh"
|
||||
|
||||
using data_t = int;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Shared-memory tiled transpose test cases (device-to-device)
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (8192,32):(1,8192), column-major
|
||||
// dst: (8192,32):(32,1), row-major
|
||||
// Shape (8192,32) creates enough 32x32 tiles to satisfy the one-wave occupancy heuristic.
|
||||
TEST_CASE("copy d2d shared_memory 2D transpose", "[copy][d2d][shared_memory][transpose]")
|
||||
{
|
||||
constexpr int M = 8192;
|
||||
constexpr int N = 32;
|
||||
constexpr int alloc = M * N;
|
||||
cuda::std::array<int, 2> shape{M, N};
|
||||
cuda::std::array<int, 2> src_strides{1, M};
|
||||
cuda::std::array<int, 2> dst_strides{N, 1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (8193,37):(1,8193), column-major
|
||||
// dst: (8193,37):(37,1), row-major
|
||||
// Extents (8193,37) are not divisible by tile size 32.
|
||||
// Boundary blocks handle the remainder.
|
||||
TEST_CASE("copy d2d shared_memory 2D partial tiles", "[copy][d2d][shared_memory][transpose][partial]")
|
||||
{
|
||||
constexpr int M = 8193;
|
||||
constexpr int N = 37;
|
||||
constexpr int alloc = M * N;
|
||||
cuda::std::array<int, 2> shape{M, N};
|
||||
cuda::std::array<int, 2> src_strides{1, M};
|
||||
cuda::std::array<int, 2> dst_strides{N, 1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (8192,16,16):(1,8192,131072), column-major
|
||||
// dst: (8192,16,16):(256,16,1), row-major
|
||||
// The simplified tensor rank remains 3, covering the generic shared-memory launch.
|
||||
TEST_CASE("copy d2d shared_memory 3D transpose", "[copy][d2d][shared_memory][transpose][3d]")
|
||||
{
|
||||
constexpr int D0 = 8192;
|
||||
constexpr int D1 = 16;
|
||||
constexpr int D2 = 16;
|
||||
constexpr int alloc = D0 * D1 * D2;
|
||||
cuda::std::array<int, 3> shape{D0, D1, D2};
|
||||
cuda::std::array<int, 3> src_strides{1, D0, D0 * D1};
|
||||
cuda::std::array<int, 3> dst_strides{D1 * D2, D2, 1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (8193,16,16):(1,8193,131088), column-major
|
||||
// dst: (8193,16,16):(256,16,1), row-major
|
||||
// The first dimension is not tile-aligned, so rank-3 boundary tiles use the direct-copy fallback.
|
||||
TEST_CASE("copy d2d shared_memory 3D partial tiles", "[copy][d2d][shared_memory][transpose][3d][partial]")
|
||||
{
|
||||
constexpr int D0 = 8193;
|
||||
constexpr int D1 = 16;
|
||||
constexpr int D2 = 16;
|
||||
constexpr int alloc = D0 * D1 * D2;
|
||||
cuda::std::array<int, 3> shape{D0, D1, D2};
|
||||
cuda::std::array<int, 3> src_strides{1, D0, D0 * D1};
|
||||
cuda::std::array<int, 3> dst_strides{D1 * D2, D2, 1};
|
||||
test_copy_stride_relaxed<data_t>(alloc, 0, shape, src_strides, alloc, 0, dst_strides);
|
||||
}
|
||||
|
||||
// src: (16,8192,8):(1,128,16)
|
||||
// dst: (16,8192,8):(131072,16,1), padded in the middle dimension.
|
||||
// This mirrors the padded small-dimension benchmark shape at unit-test scale.
|
||||
TEST_CASE("copy d2d shared_memory 3D padded small dimension", "[copy][d2d][shared_memory][transpose][3d][padded]")
|
||||
{
|
||||
constexpr int D0 = 16;
|
||||
constexpr int D1 = 8192;
|
||||
constexpr int D2 = 8;
|
||||
constexpr int dst_pitch = 16;
|
||||
constexpr int src_alloc = D0 * D1 * D2;
|
||||
constexpr int dst_alloc = D0 * D1 * dst_pitch;
|
||||
cuda::std::array<int, 3> shape{D0, D1, D2};
|
||||
cuda::std::array<int, 3> src_strides{1, D0 * D2, D0};
|
||||
cuda::std::array<int, 3> dst_strides{D1 * dst_pitch, dst_pitch, 1};
|
||||
test_copy_stride_relaxed<data_t>(src_alloc, 0, shape, src_strides, dst_alloc, 0, dst_strides);
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "copy_common.cuh"
|
||||
|
||||
// All layouts in this file have 48 contiguous elements that should coalesce into a (48):(1) layout
|
||||
// and generate 16B vectorized copies.
|
||||
|
||||
static constexpr int N = 48;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* 1D Vectorization Test
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (48):(1)
|
||||
// dst: (48):(1)
|
||||
TEST_CASE("copy d2d vectorize 48:1", "[copy][d2d][vectorize][1d]")
|
||||
{
|
||||
test_copy<layout_right>(make_iota<int>(N), N);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* 2D Vectorization Tests
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (6,8):(8,1)
|
||||
// dst: (6,8):(8,1)
|
||||
TEST_CASE("copy d2d vectorize (6,8):(8,1)", "[copy][d2d][vectorize][2d]")
|
||||
{
|
||||
test_copy<layout_right>(make_iota<int>(N), 6, 8);
|
||||
}
|
||||
|
||||
// src: (8,6):(1,8)
|
||||
// dst: (8,6):(1,8)
|
||||
TEST_CASE("copy d2d vectorize (8,6):(1,8)", "[copy][d2d][vectorize][2d]")
|
||||
{
|
||||
test_copy<layout_left>(make_iota<int>(N), 8, 6);
|
||||
}
|
||||
|
||||
// src: (6,8):(1,6)
|
||||
// dst: (6,8):(1,6)
|
||||
TEST_CASE("copy d2d vectorize (6,8):(1,6)", "[copy][d2d][vectorize][2d]")
|
||||
{
|
||||
test_copy<layout_left>(make_iota<int>(N), 6, 8);
|
||||
}
|
||||
|
||||
// src: (8,6):(6,1)
|
||||
// dst: (8,6):(6,1)
|
||||
TEST_CASE("copy d2d vectorize (8,6):(6,1)", "[copy][d2d][vectorize][2d]")
|
||||
{
|
||||
test_copy<layout_right>(make_iota<int>(N), 8, 6);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* 2D Non-Square Vectorization Tests
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (3,16):(1,3)
|
||||
// dst: (3,16):(1,3)
|
||||
TEST_CASE("copy d2d vectorize (3,16):(1,3)", "[copy][d2d][vectorize][2d]")
|
||||
{
|
||||
test_copy<layout_left>(make_iota<int>(N), 3, 16);
|
||||
}
|
||||
|
||||
// src: (3,16):(16,1)
|
||||
// dst: (3,16):(16,1)
|
||||
TEST_CASE("copy d2d vectorize (3,16):(16,1)", "[copy][d2d][vectorize][2d]")
|
||||
{
|
||||
test_copy<layout_right>(make_iota<int>(N), 3, 16);
|
||||
}
|
||||
|
||||
// src: (16,3):(1,16)
|
||||
// dst: (16,3):(1,16)
|
||||
TEST_CASE("copy d2d vectorize (16,3):(1,16)", "[copy][d2d][vectorize][2d]")
|
||||
{
|
||||
test_copy<layout_left>(make_iota<int>(N), 16, 3);
|
||||
}
|
||||
|
||||
// src: (16,3):(3,1)
|
||||
// dst: (16,3):(3,1)
|
||||
TEST_CASE("copy d2d vectorize (16,3):(3,1)", "[copy][d2d][vectorize][2d]")
|
||||
{
|
||||
test_copy<layout_right>(make_iota<int>(N), 16, 3);
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
// 5D vectorization tests (3 tests) in a separate TU to avoid a GCC false-positive -Warray-bounds warning when rank-2
|
||||
// and rank-5 template instantiations coexist
|
||||
|
||||
#include "copy_common.cuh"
|
||||
|
||||
// 5D layouts with 48 contiguous elements that should coalesce into a (48):(1) layout
|
||||
// and generate 16B vectorized copies.
|
||||
|
||||
static constexpr int N = 48;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* 5D Vectorization Tests
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (2,2,2,2,3):(2,1,4,8,16)
|
||||
// dst: (2,2,2,2,3):(2,1,4,8,16)
|
||||
TEST_CASE("copy d2d vectorize (2,2,2,2,3):(2,1,4,8,16)", "[copy][d2d][vectorize][5d]")
|
||||
{
|
||||
test_copy_strided(
|
||||
make_iota<int>(N), cuda::std::array<int, 5>{2, 2, 2, 2, 3}, cuda::std::array<int, 5>{2, 1, 4, 8, 16});
|
||||
}
|
||||
|
||||
// src: (2,2,2,2,3):(8,1,4,2,16)
|
||||
// dst: (2,2,2,2,3):(8,1,4,2,16)
|
||||
TEST_CASE("copy d2d vectorize (2,2,2,2,3):(8,1,4,2,16)", "[copy][d2d][vectorize][5d]")
|
||||
{
|
||||
test_copy_strided(
|
||||
make_iota<int>(N), cuda::std::array<int, 5>{2, 2, 2, 2, 3}, cuda::std::array<int, 5>{8, 1, 4, 2, 16});
|
||||
}
|
||||
|
||||
// src: (2,2,3,2,2):(6,3,1,24,12)
|
||||
// dst: (2,2,3,2,2):(6,3,1,24,12)
|
||||
TEST_CASE("copy d2d vectorize (2,2,3,2,2):(6,3,1,24,12)", "[copy][d2d][vectorize][5d]")
|
||||
{
|
||||
test_copy_strided(
|
||||
make_iota<int>(N), cuda::std::array<int, 5>{2, 2, 3, 2, 2}, cuda::std::array<int, 5>{6, 3, 1, 24, 12});
|
||||
}
|
||||
Reference in New Issue
Block a user