[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
79
cccl_upstream/c/experimental/stf/CMakeLists.txt
Normal file
79
cccl_upstream/c/experimental/stf/CMakeLists.txt
Normal file
@@ -0,0 +1,79 @@
|
||||
cmake_minimum_required(VERSION 3.21)
|
||||
|
||||
project(CCCL_C_EXPERIMENTAL_STF LANGUAGES CUDA CXX C)
|
||||
|
||||
option(
|
||||
CCCL_C_EXPERIMENTAL_STF_ENABLE_TESTING
|
||||
"Build cccl.experimental.c.stf tests."
|
||||
OFF
|
||||
)
|
||||
|
||||
# FIXME Ideally this would be handled by presets and install rules, but for now
|
||||
# consumers may override this to control the target location of cccl.c.experimental.stf.
|
||||
set(
|
||||
CCCL_C_EXPERIMENTAL_STF_LIBRARY_OUTPUT_DIRECTORY
|
||||
""
|
||||
CACHE PATH
|
||||
"Override output directory for the cccl.c.experimental.stf library"
|
||||
)
|
||||
mark_as_advanced(CCCL_C_EXPERIMENTAL_STF_LIBRARY_OUTPUT_DIRECTORY)
|
||||
|
||||
file(
|
||||
GLOB_RECURSE srcs
|
||||
RELATIVE "${CMAKE_CURRENT_LIST_DIR}"
|
||||
CONFIGURE_DEPENDS
|
||||
"src/*.cu"
|
||||
"src/*.cuh"
|
||||
)
|
||||
|
||||
cccl_get_cudatoolkit()
|
||||
cccl_get_cudax()
|
||||
|
||||
add_library(cccl.c.experimental.stf SHARED ${srcs})
|
||||
set_property(
|
||||
TARGET cccl.c.experimental.stf
|
||||
PROPERTY POSITION_INDEPENDENT_CODE ON
|
||||
)
|
||||
cccl_configure_target(cccl.c.experimental.stf DIALECT 20)
|
||||
|
||||
# Override the properties set by cccl_configure_target:
|
||||
if (CCCL_C_EXPERIMENTAL_STF_LIBRARY_OUTPUT_DIRECTORY)
|
||||
set_target_properties(
|
||||
cccl.c.experimental.stf
|
||||
PROPERTIES
|
||||
LIBRARY_OUTPUT_DIRECTORY
|
||||
"${CCCL_C_EXPERIMENTAL_STF_LIBRARY_OUTPUT_DIRECTORY}"
|
||||
ARCHIVE_OUTPUT_DIRECTORY
|
||||
"${CCCL_C_EXPERIMENTAL_STF_LIBRARY_OUTPUT_DIRECTORY}"
|
||||
)
|
||||
endif()
|
||||
|
||||
set_target_properties(
|
||||
cccl.c.experimental.stf
|
||||
PROPERTIES CUDA_RUNTIME_LIBRARY STATIC
|
||||
)
|
||||
target_compile_definitions(cccl.c.experimental.stf PUBLIC CCCL_C_EXPERIMENTAL=1)
|
||||
target_link_libraries(
|
||||
cccl.c.experimental.stf
|
||||
PRIVATE #
|
||||
CUDA::cudart_static
|
||||
CUDA::cuda_driver
|
||||
cudax::cudax
|
||||
)
|
||||
|
||||
target_compile_options(
|
||||
cccl.c.experimental.stf
|
||||
PRIVATE #
|
||||
$<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:--expt-relaxed-constexpr>
|
||||
$<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:--extended-lambda>
|
||||
)
|
||||
|
||||
target_include_directories(
|
||||
cccl.c.experimental.stf
|
||||
PUBLIC "include"
|
||||
PRIVATE "src"
|
||||
)
|
||||
|
||||
if (CCCL_C_EXPERIMENTAL_STF_ENABLE_TESTING)
|
||||
add_subdirectory(test)
|
||||
endif()
|
||||
File diff suppressed because it is too large
Load Diff
1938
cccl_upstream/c/experimental/stf/src/stf.cu
Normal file
1938
cccl_upstream/c/experimental/stf/src/stf.cu
Normal file
File diff suppressed because it is too large
Load Diff
44
cccl_upstream/c/experimental/stf/test/CMakeLists.txt
Normal file
44
cccl_upstream/c/experimental/stf/test/CMakeLists.txt
Normal file
@@ -0,0 +1,44 @@
|
||||
cccl_get_c2h()
|
||||
|
||||
function(cccl_c_experimental_stf_add_test target_name_var source)
|
||||
string(
|
||||
REGEX REPLACE
|
||||
"test_([^.]*)"
|
||||
"cccl.c.experimental.stf.test.\\1"
|
||||
target_name
|
||||
"${source}"
|
||||
)
|
||||
set(target_name_var ${target_name} PARENT_SCOPE)
|
||||
|
||||
cccl_add_executable(
|
||||
${target_name}
|
||||
ADD_CTEST
|
||||
NO_METATARGETS
|
||||
DIALECT 20
|
||||
SOURCES "${source}"
|
||||
)
|
||||
|
||||
set_target_properties(${target_name} PROPERTIES CUDA_RUNTIME_LIBRARY STATIC)
|
||||
target_link_libraries(
|
||||
${target_name}
|
||||
PRIVATE
|
||||
cccl.compiler_interface
|
||||
cccl.c.experimental.stf
|
||||
CUDA::cudart_static
|
||||
CUDA::nvrtc
|
||||
cccl.c2h.main
|
||||
CUDA::cuda_driver
|
||||
)
|
||||
endfunction()
|
||||
|
||||
file(
|
||||
GLOB test_srcs
|
||||
RELATIVE "${CMAKE_CURRENT_LIST_DIR}"
|
||||
CONFIGURE_DEPENDS
|
||||
*.cu
|
||||
*.cpp
|
||||
)
|
||||
|
||||
foreach (test_src IN LISTS test_srcs)
|
||||
cccl_c_experimental_stf_add_test(test_target "${test_src}")
|
||||
endforeach()
|
||||
163
cccl_upstream/c/experimental/stf/test/test_allocate_nd.cpp
Normal file
163
cccl_upstream/c/experimental/stf/test/test_allocate_nd.cpp
Normal file
@@ -0,0 +1,163 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cstdint>
|
||||
#include <vector>
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <cccl/c/experimental/stf/stf.h>
|
||||
|
||||
namespace
|
||||
{
|
||||
inline constexpr uint64_t one_mib = 1024 * 1024;
|
||||
|
||||
stf_exec_place_handle make_dev0_grid(size_t nplaces)
|
||||
{
|
||||
std::vector<stf_exec_place_handle> places(nplaces);
|
||||
for (auto& place : places)
|
||||
{
|
||||
place = stf_exec_place_device(0);
|
||||
REQUIRE(place != nullptr);
|
||||
}
|
||||
stf_exec_place_handle const grid = stf_exec_place_grid_create(places.data(), nplaces, nullptr);
|
||||
REQUIRE(grid != nullptr);
|
||||
for (const auto& place : places)
|
||||
{
|
||||
stf_exec_place_destroy(place);
|
||||
}
|
||||
return grid;
|
||||
}
|
||||
|
||||
void check_device_round_trip(void* ptr, uint64_t n)
|
||||
{
|
||||
const std::vector<int> host(n, 42);
|
||||
REQUIRE(cudaMemcpy(ptr, host.data(), n * sizeof(int), cudaMemcpyHostToDevice) == cudaSuccess);
|
||||
std::vector<int> back(n, 0);
|
||||
REQUIRE(cudaMemcpy(back.data(), ptr, n * sizeof(int), cudaMemcpyDeviceToHost) == cudaSuccess);
|
||||
REQUIRE(back[0] == 42);
|
||||
REQUIRE(back[n - 1] == 42);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("shaped allocation on an ordinary data place", "[places][allocate]")
|
||||
{
|
||||
constexpr uint64_t n = one_mib; // ints
|
||||
constexpr stf_dim4 dims{n, 1, 1, 1};
|
||||
|
||||
stf_data_place_handle const dp = stf_data_place_device(0);
|
||||
REQUIRE(dp != nullptr);
|
||||
|
||||
// On a non-composite place the geometry degenerates to a byte count
|
||||
void* const ptr = stf_data_place_allocate_nd(dp, &dims, sizeof(int), nullptr);
|
||||
REQUIRE(ptr != nullptr);
|
||||
check_device_round_trip(ptr, n);
|
||||
|
||||
stf_data_place_deallocate(dp, ptr, n * sizeof(int), nullptr);
|
||||
stf_data_place_destroy(dp);
|
||||
}
|
||||
|
||||
C2H_TEST("shaped allocation on composite data places", "[places][allocate]")
|
||||
{
|
||||
stf_exec_place_handle const grid = make_dev0_grid(2);
|
||||
|
||||
constexpr uint64_t n = one_mib; // ints
|
||||
constexpr stf_dim4 dims{n, 1, 1, 1};
|
||||
|
||||
stf_data_place_handle const dp = stf_data_place_composite(grid, stf_partition_fn_blocked(0));
|
||||
REQUIRE(dp != nullptr);
|
||||
|
||||
// A byte count alone cannot carry the tensor geometry: must fail cleanly
|
||||
void* const bad = stf_data_place_allocate(dp, static_cast<ptrdiff_t>(n * sizeof(int)), nullptr);
|
||||
REQUIRE(bad == nullptr);
|
||||
|
||||
void* const ptr = stf_data_place_allocate_nd(dp, &dims, sizeof(int), nullptr);
|
||||
REQUIRE(ptr != nullptr);
|
||||
|
||||
// Memory must be usable from the device
|
||||
check_device_round_trip(ptr, n);
|
||||
|
||||
stf_data_place_deallocate(dp, ptr, n * sizeof(int), nullptr);
|
||||
stf_data_place_destroy(dp);
|
||||
|
||||
// Same flow through the native cyclic partition function
|
||||
stf_data_place_handle const dpc = stf_data_place_composite(grid, stf_partition_fn_cyclic());
|
||||
REQUIRE(dpc != nullptr);
|
||||
|
||||
void* const ptr2 = stf_data_place_allocate_nd(dpc, &dims, sizeof(int), nullptr);
|
||||
REQUIRE(ptr2 != nullptr);
|
||||
check_device_round_trip(ptr2, n);
|
||||
|
||||
stf_data_place_deallocate(dpc, ptr2, n * sizeof(int), nullptr);
|
||||
stf_data_place_destroy(dpc);
|
||||
stf_exec_place_destroy(grid);
|
||||
}
|
||||
|
||||
C2H_TEST("blocked partition function covers every dimension selector", "[places][allocate]")
|
||||
{
|
||||
stf_exec_place_handle const grid = make_dev0_grid(2);
|
||||
|
||||
// 64 * 64 * 16 * 4 ints = 1 MiB: every dimension is divisible by the grid
|
||||
constexpr stf_dim4 dims{64, 64, 16, 4};
|
||||
constexpr uint64_t n = dims.x * dims.y * dims.z * dims.t;
|
||||
|
||||
// Dimensions 0-3 select that axis; out-of-range values (like -1) select the
|
||||
// highest axis whose extent is greater than one. All must yield a usable
|
||||
// native mapper.
|
||||
for (const int dim : {0, 1, 2, 3, -1, 4})
|
||||
{
|
||||
const stf_get_executor_fn mapper = stf_partition_fn_blocked(dim);
|
||||
REQUIRE(mapper != nullptr);
|
||||
|
||||
stf_data_place_handle const dp = stf_data_place_composite(grid, mapper);
|
||||
REQUIRE(dp != nullptr);
|
||||
|
||||
void* const ptr = stf_data_place_allocate_nd(dp, &dims, sizeof(int), nullptr);
|
||||
REQUIRE(ptr != nullptr);
|
||||
check_device_round_trip(ptr, n);
|
||||
|
||||
stf_data_place_deallocate(dp, ptr, n * sizeof(int), nullptr);
|
||||
stf_data_place_destroy(dp);
|
||||
}
|
||||
|
||||
stf_exec_place_destroy(grid);
|
||||
}
|
||||
|
||||
C2H_TEST("shaped allocation rejects overflowing geometries", "[places][allocate]")
|
||||
{
|
||||
// (2^64-1)^2 wraps to 1: an unchecked size computation would hand back a
|
||||
// one-byte allocation for an astronomically large tensor
|
||||
constexpr stf_dim4 huge{UINT64_MAX, UINT64_MAX, 1, 1};
|
||||
|
||||
stf_data_place_handle const dp = stf_data_place_device(0);
|
||||
REQUIRE(dp != nullptr);
|
||||
REQUIRE(stf_data_place_allocate_nd(dp, &huge, 1, nullptr) == nullptr);
|
||||
|
||||
// elemsize participates in the product too
|
||||
constexpr stf_dim4 max_1d{UINT64_MAX, 1, 1, 1};
|
||||
REQUIRE(stf_data_place_allocate_nd(dp, &max_1d, 2, nullptr) == nullptr);
|
||||
|
||||
// A representable product that exceeds PTRDIFF_MAX must also be rejected
|
||||
constexpr stf_dim4 above_ptrdiff{uint64_t{1} << 62, 2, 1, 1};
|
||||
REQUIRE(stf_data_place_allocate_nd(dp, &above_ptrdiff, 1, nullptr) == nullptr);
|
||||
|
||||
stf_data_place_destroy(dp);
|
||||
|
||||
// On a composite place the wrapped geometry used to reach the blocked
|
||||
// partitioner with a zero part_size and kill the process with SIGFPE
|
||||
stf_exec_place_handle const grid = make_dev0_grid(2);
|
||||
stf_data_place_handle const dpc = stf_data_place_composite(grid, stf_partition_fn_blocked(1));
|
||||
REQUIRE(dpc != nullptr);
|
||||
REQUIRE(stf_data_place_allocate_nd(dpc, &huge, 1, nullptr) == nullptr);
|
||||
|
||||
stf_data_place_destroy(dpc);
|
||||
stf_exec_place_destroy(grid);
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
// Focused tests for stf_async_resources_create/destroy() exercised through
|
||||
// stf_ctx_create_ex(). They cover the contract documented in stf.h:
|
||||
//
|
||||
// * A shared stf_async_resources_handle can be reused across multiple
|
||||
// contexts created via stf_ctx_create_ex().
|
||||
// * When the contexts are created with `has_stream = 1`, stf_ctx_finalize()
|
||||
// is non-blocking: the caller must cudaStreamSynchronize(user_stream)
|
||||
// before destroying the shared handle.
|
||||
//
|
||||
// Both backends (STF_BACKEND_STREAM and STF_BACKEND_GRAPH) are exercised
|
||||
// because the graph backend additionally benefits from the handle's
|
||||
// executable-graph cache.
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <cccl/c/experimental/stf/stf.h>
|
||||
|
||||
// A device sink that is written but never read. Publishing the busy-loop
|
||||
// result here gives the loop an observable side effect, so the compiler
|
||||
// cannot optimize it away, without perturbing the result buffer.
|
||||
__device__ unsigned g_busy_sink;
|
||||
|
||||
// Writes `value` into every slot of `arr`. The inner busy loop widens the
|
||||
// kernel window so a missing chain dependency between back-to-back contexts
|
||||
// becomes observable: a slow ctx1 kernel must finish before ctx2's kernel
|
||||
// commits its value.
|
||||
__global__ void slow_set_kernel(int* arr, int n, int value, int iters)
|
||||
{
|
||||
const int tid = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
|
||||
if (tid >= n)
|
||||
{
|
||||
return;
|
||||
}
|
||||
// Busy loop to keep the kernel resident on the SM for a while. `acc` is
|
||||
// unsigned so the accumulation wraps with well-defined behavior.
|
||||
unsigned acc = 0;
|
||||
for (int i = 0; i < iters; ++i)
|
||||
{
|
||||
acc += (static_cast<unsigned>(i) * 1103515245u + 12345u) & 0x7fffffffu;
|
||||
}
|
||||
// Publish `acc` via an atomic: an observable, race-free side effect that
|
||||
// keeps the loop alive while the stored result stays exactly `value`.
|
||||
atomicAdd(&g_busy_sink, acc);
|
||||
arr[tid] = value;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
// Submit one slow_set kernel into `ctx`, writing `value` everywhere in
|
||||
// `d_arr`. Use stf_cuda_kernel_* instead of the generic task stream API so
|
||||
// this helper is valid for both stream and graph backends.
|
||||
void submit_set_kernel(stf_ctx_handle ctx, int* d_arr, int n, int value, int iters)
|
||||
{
|
||||
int dev_id = 0;
|
||||
REQUIRE(cudaGetDevice(&dev_id) == cudaSuccess);
|
||||
stf_data_place_handle dev_place = stf_data_place_device(dev_id);
|
||||
stf_logical_data_handle lD = stf_logical_data_with_place(ctx, d_arr, n * sizeof(int), dev_place);
|
||||
REQUIRE(lD != nullptr);
|
||||
stf_data_place_destroy(dev_place);
|
||||
stf_logical_data_set_symbol(lD, "device_buffer");
|
||||
|
||||
stf_cuda_kernel_handle k = stf_cuda_kernel_create(ctx);
|
||||
REQUIRE(k != nullptr);
|
||||
stf_cuda_kernel_set_symbol(k, "slow_set");
|
||||
stf_cuda_kernel_add_dep(k, lD, STF_RW);
|
||||
stf_cuda_kernel_start(k);
|
||||
|
||||
int* arg_ptr = static_cast<int*>(stf_cuda_kernel_get_arg(k, 0));
|
||||
REQUIRE(arg_ptr == d_arr);
|
||||
const int threads = 128;
|
||||
const int blocks = (n + threads - 1) / threads;
|
||||
const void* args[4] = {&arg_ptr, &n, &value, &iters};
|
||||
cudaError_t err =
|
||||
stf_cuda_kernel_add_desc(k, reinterpret_cast<void*>(slow_set_kernel), dim3(blocks), dim3(threads), 0, 4, args);
|
||||
REQUIRE(err == cudaSuccess);
|
||||
stf_cuda_kernel_end(k);
|
||||
stf_cuda_kernel_destroy(k);
|
||||
|
||||
stf_logical_data_destroy(lD);
|
||||
}
|
||||
|
||||
// Run one ctx (created via stf_ctx_create_ex with a caller-provided stream
|
||||
// and a shared async_resources handle) that issues a single slow_set kernel.
|
||||
void run_ctx_with_handle(
|
||||
stf_backend_kind backend, cudaStream_t s, stf_async_resources_handle h, int* d_arr, int N, int value, int iters)
|
||||
{
|
||||
stf_ctx_options opts{};
|
||||
opts.backend = backend;
|
||||
opts.has_stream = 1;
|
||||
opts.stream = s;
|
||||
opts.handle = h;
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create_ex(&opts);
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
submit_set_kernel(ctx, d_arr, N, value, iters);
|
||||
|
||||
// Non-blocking: this enqueues the remaining work and the resource-release
|
||||
// callback on `s`; it does not synchronize `s`.
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
// Run a back-to-back ordering experiment: two contexts share a handle and a
|
||||
// caller stream, write distinct values, and the final buffer must reflect
|
||||
// the second context's value (ctx2-after-ctx1 ordering via the caller
|
||||
// stream). Iterating amplifies any missed dependency.
|
||||
void check_back_to_back_ordering(stf_backend_kind backend)
|
||||
{
|
||||
constexpr int N = 1 << 14;
|
||||
constexpr int ITERS = 1 << 18;
|
||||
|
||||
cudaStream_t s{};
|
||||
REQUIRE(cudaStreamCreate(&s) == cudaSuccess);
|
||||
|
||||
int* d_arr = nullptr;
|
||||
REQUIRE(cudaMalloc(&d_arr, N * sizeof(int)) == cudaSuccess);
|
||||
REQUIRE(cudaMemsetAsync(d_arr, 0, N * sizeof(int), s) == cudaSuccess);
|
||||
|
||||
stf_async_resources_handle h = stf_async_resources_create();
|
||||
REQUIRE(h != nullptr);
|
||||
|
||||
for (int iter = 0; iter < 20; ++iter)
|
||||
{
|
||||
run_ctx_with_handle(backend, s, h, d_arr, N, /*value=*/1, ITERS);
|
||||
run_ctx_with_handle(backend, s, h, d_arr, N, /*value=*/2, ITERS);
|
||||
|
||||
REQUIRE(cudaStreamSynchronize(s) == cudaSuccess);
|
||||
int h_arr[16]{};
|
||||
REQUIRE(cudaMemcpy(h_arr, d_arr, sizeof(h_arr), cudaMemcpyDeviceToHost) == cudaSuccess);
|
||||
for (int i = 0; i < static_cast<int>(sizeof(h_arr) / sizeof(int)); ++i)
|
||||
{
|
||||
INFO("iter=" << iter << " i=" << i << " value=" << h_arr[i]);
|
||||
REQUIRE(h_arr[i] == 2);
|
||||
}
|
||||
}
|
||||
|
||||
// Required before destroying `h`: stf_ctx_finalize() left resource-release
|
||||
// callbacks enqueued on `s`. The destroy call tears down the underlying
|
||||
// CUDA resources synchronously.
|
||||
REQUIRE(cudaStreamSynchronize(s) == cudaSuccess);
|
||||
stf_async_resources_destroy(h);
|
||||
|
||||
REQUIRE(cudaFree(d_arr) == cudaSuccess);
|
||||
REQUIRE(cudaStreamDestroy(s) == cudaSuccess);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("stf_async_resources_handle: shared across back-to-back stream contexts on user stream",
|
||||
"[context][stream][async_resources_handle]")
|
||||
{
|
||||
check_back_to_back_ordering(STF_BACKEND_STREAM);
|
||||
}
|
||||
|
||||
C2H_TEST("stf_async_resources_handle: shared across back-to-back graph contexts on user stream",
|
||||
"[context][graph][async_resources_handle]")
|
||||
{
|
||||
check_back_to_back_ordering(STF_BACKEND_GRAPH);
|
||||
}
|
||||
|
||||
// Smoke check of the handle's lifetime API independent of any context:
|
||||
// * NULL is a no-op for stf_async_resources_destroy().
|
||||
// * Create/destroy without ever attaching the handle to a context works.
|
||||
// * Destroying the handle before having submitted any work via a context
|
||||
// (only after a context was created and finalized without tasks) is
|
||||
// safe.
|
||||
C2H_TEST("stf_async_resources_handle: lifetime smoke (no work, NULL destroy)",
|
||||
"[context][async_resources_handle][lifetime]")
|
||||
{
|
||||
stf_async_resources_destroy(nullptr);
|
||||
|
||||
stf_async_resources_handle h = stf_async_resources_create();
|
||||
REQUIRE(h != nullptr);
|
||||
stf_async_resources_destroy(h);
|
||||
|
||||
// Empty stream context bound to a user stream + handle, no submitted work.
|
||||
cudaStream_t s{};
|
||||
REQUIRE(cudaStreamCreate(&s) == cudaSuccess);
|
||||
|
||||
stf_async_resources_handle h2 = stf_async_resources_create();
|
||||
REQUIRE(h2 != nullptr);
|
||||
{
|
||||
stf_ctx_options opts{};
|
||||
opts.backend = STF_BACKEND_STREAM;
|
||||
opts.has_stream = 1;
|
||||
opts.stream = s;
|
||||
opts.handle = h2;
|
||||
stf_ctx_handle ctx = stf_ctx_create_ex(&opts);
|
||||
REQUIRE(ctx != nullptr);
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
// Even without submitted tasks the finalize is non-blocking when the
|
||||
// context was created with `has_stream = 1`. Synchronize before destroy.
|
||||
REQUIRE(cudaStreamSynchronize(s) == cudaSuccess);
|
||||
stf_async_resources_destroy(h2);
|
||||
|
||||
REQUIRE(cudaStreamDestroy(s) == cudaSuccess);
|
||||
}
|
||||
92
cccl_upstream/c/experimental/stf/test/test_ctx.cpp
Normal file
92
cccl_upstream/c/experimental/stf/test/test_ctx.cpp
Normal file
@@ -0,0 +1,92 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <cccl/c/experimental/stf/stf.h>
|
||||
|
||||
C2H_TEST("basic stf context", "[context]")
|
||||
{
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
C2H_TEST("stf_ctx_wait reads data without finalizing", "[context]")
|
||||
{
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
int h_value = 0;
|
||||
stf_logical_data_handle lVal = stf_logical_data(ctx, &h_value, sizeof(int));
|
||||
REQUIRE(lVal != nullptr);
|
||||
stf_logical_data_set_symbol(lVal, "val");
|
||||
|
||||
int src_val = 42;
|
||||
|
||||
stf_host_launch_handle h = stf_host_launch_create(ctx);
|
||||
REQUIRE(h != nullptr);
|
||||
stf_host_launch_set_symbol(h, "set42");
|
||||
stf_host_launch_add_dep(h, lVal, STF_WRITE);
|
||||
stf_host_launch_set_user_data(h, &src_val, sizeof(int), nullptr);
|
||||
stf_host_launch_submit(h, [](stf_host_launch_deps_handle deps) {
|
||||
int* data = (int*) stf_host_launch_deps_get(deps, 0);
|
||||
int* src = (int*) stf_host_launch_deps_get_user_data(deps);
|
||||
data[0] = *src;
|
||||
});
|
||||
stf_host_launch_destroy(h);
|
||||
|
||||
int result = 0;
|
||||
int rc = stf_ctx_wait(ctx, lVal, &result, sizeof(int));
|
||||
REQUIRE(rc == 0);
|
||||
REQUIRE(result == 42);
|
||||
|
||||
// The context remains usable after waiting.
|
||||
src_val = 99;
|
||||
|
||||
stf_host_launch_handle h2 = stf_host_launch_create(ctx);
|
||||
REQUIRE(h2 != nullptr);
|
||||
stf_host_launch_set_symbol(h2, "set99");
|
||||
stf_host_launch_add_dep(h2, lVal, STF_WRITE);
|
||||
stf_host_launch_set_user_data(h2, &src_val, sizeof(int), nullptr);
|
||||
stf_host_launch_submit(h2, [](stf_host_launch_deps_handle deps) {
|
||||
int* data = (int*) stf_host_launch_deps_get(deps, 0);
|
||||
int* src = (int*) stf_host_launch_deps_get_user_data(deps);
|
||||
data[0] = *src;
|
||||
});
|
||||
stf_host_launch_destroy(h2);
|
||||
|
||||
result = 0;
|
||||
rc = stf_ctx_wait(ctx, lVal, &result, sizeof(int));
|
||||
REQUIRE(rc == 0);
|
||||
REQUIRE(result == 99);
|
||||
|
||||
stf_logical_data_destroy(lVal);
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
C2H_TEST("stf_ctx_wait rejects invalid arguments", "[context]")
|
||||
{
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
int h_value = 0;
|
||||
stf_logical_data_handle lVal = stf_logical_data(ctx, &h_value, sizeof(int));
|
||||
REQUIRE(lVal != nullptr);
|
||||
|
||||
int result = 0;
|
||||
REQUIRE(stf_ctx_wait(nullptr, lVal, &result, sizeof(int)) != 0);
|
||||
REQUIRE(stf_ctx_wait(ctx, nullptr, &result, sizeof(int)) != 0);
|
||||
REQUIRE(stf_ctx_wait(ctx, lVal, nullptr, sizeof(int)) != 0);
|
||||
|
||||
stf_logical_data_destroy(lVal);
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
89
cccl_upstream/c/experimental/stf/test/test_cuda_kernel.cu
Normal file
89
cccl_upstream/c/experimental/stf/test/test_cuda_kernel.cu
Normal file
@@ -0,0 +1,89 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <cccl/c/experimental/stf/stf.h>
|
||||
|
||||
__global__ void axpy(int cnt, double a, const double* x, double* y)
|
||||
{
|
||||
int tid = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
|
||||
int nthreads = static_cast<int>(gridDim.x * blockDim.x);
|
||||
|
||||
for (int i = tid; i < cnt; i += nthreads)
|
||||
{
|
||||
y[i] += a * x[i];
|
||||
}
|
||||
}
|
||||
|
||||
double X0(int i)
|
||||
{
|
||||
return sin(static_cast<double>(i));
|
||||
}
|
||||
|
||||
double Y0(int i)
|
||||
{
|
||||
return cos((double) i);
|
||||
}
|
||||
|
||||
C2H_TEST("axpy with stf cuda_kernel", "[cuda_kernel]")
|
||||
{
|
||||
size_t N = 1000000;
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
std::vector<double> X(N);
|
||||
std::vector<double> Y(N);
|
||||
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
X[i] = X0(static_cast<int>(i));
|
||||
Y[i] = Y0(static_cast<int>(i));
|
||||
}
|
||||
|
||||
const double alpha = 3.14;
|
||||
|
||||
stf_logical_data_handle lX = stf_logical_data(ctx, X.data(), N * sizeof(double));
|
||||
stf_logical_data_handle lY = stf_logical_data(ctx, Y.data(), N * sizeof(double));
|
||||
REQUIRE(lX != nullptr);
|
||||
REQUIRE(lY != nullptr);
|
||||
|
||||
stf_logical_data_set_symbol(lX, "X");
|
||||
stf_logical_data_set_symbol(lY, "Y");
|
||||
|
||||
stf_cuda_kernel_handle k = stf_cuda_kernel_create(ctx);
|
||||
REQUIRE(k != nullptr);
|
||||
stf_cuda_kernel_set_symbol(k, "axpy");
|
||||
stf_cuda_kernel_add_dep(k, lX, STF_READ);
|
||||
stf_cuda_kernel_add_dep(k, lY, STF_RW);
|
||||
stf_cuda_kernel_start(k);
|
||||
double* dX = (double*) stf_cuda_kernel_get_arg(k, 0);
|
||||
double* dY = (double*) stf_cuda_kernel_get_arg(k, 1);
|
||||
const void* args[4] = {&N, &alpha, &dX, &dY};
|
||||
cudaError_t err = stf_cuda_kernel_add_desc(k, (void*) axpy, 2, 4, 0, 4, args);
|
||||
REQUIRE(err == cudaSuccess);
|
||||
stf_cuda_kernel_end(k);
|
||||
stf_cuda_kernel_destroy(k);
|
||||
|
||||
stf_logical_data_destroy(lX);
|
||||
stf_logical_data_destroy(lY);
|
||||
|
||||
stf_ctx_finalize(ctx);
|
||||
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
assert(fabs(Y[i] - (Y0(i) + alpha * X0(i))) < 0.0001);
|
||||
assert(fabs(X[i] - X0(i)) < 0.0001);
|
||||
}
|
||||
}
|
||||
263
cccl_upstream/c/experimental/stf/test/test_host_launch.cu
Normal file
263
cccl_upstream/c/experimental/stf/test/test_host_launch.cu
Normal file
@@ -0,0 +1,263 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cmath>
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <cccl/c/experimental/stf/stf.h>
|
||||
|
||||
__global__ void fill_kernel(int cnt, double* data, double value)
|
||||
{
|
||||
int tid = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
|
||||
int nthreads = static_cast<int>(gridDim.x * blockDim.x);
|
||||
|
||||
for (int i = tid; i < cnt; i += nthreads)
|
||||
{
|
||||
data[i] = value + i;
|
||||
}
|
||||
}
|
||||
|
||||
struct verify_args
|
||||
{
|
||||
size_t N;
|
||||
bool* passed;
|
||||
};
|
||||
|
||||
static void verify_callback(stf_host_launch_deps_handle deps)
|
||||
{
|
||||
auto* v = static_cast<verify_args*>(stf_host_launch_deps_get_user_data(deps));
|
||||
|
||||
if (stf_host_launch_deps_size(deps) != 1)
|
||||
{
|
||||
*v->passed = false;
|
||||
return;
|
||||
}
|
||||
|
||||
if (stf_host_launch_deps_get_size(deps, 0) != v->N * sizeof(double))
|
||||
{
|
||||
*v->passed = false;
|
||||
return;
|
||||
}
|
||||
|
||||
auto* data = static_cast<double*>(stf_host_launch_deps_get(deps, 0));
|
||||
for (size_t i = 0; i < v->N; i++)
|
||||
{
|
||||
if (fabs(data[i] - (42.0 + static_cast<double>(i))) > 1e-10)
|
||||
{
|
||||
*v->passed = false;
|
||||
return;
|
||||
}
|
||||
}
|
||||
*v->passed = true;
|
||||
}
|
||||
|
||||
C2H_TEST("host_launch with stream context", "[host_launch]")
|
||||
{
|
||||
const size_t N = 1024;
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
double* host_data;
|
||||
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
host_data[i] = 0.0;
|
||||
}
|
||||
|
||||
stf_logical_data_handle lData = stf_logical_data(ctx, host_data, N * sizeof(double));
|
||||
REQUIRE(lData != nullptr);
|
||||
stf_logical_data_set_symbol(lData, "data");
|
||||
|
||||
// Fill data via a kernel task
|
||||
stf_task_handle t = stf_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_task_set_symbol(t, "fill");
|
||||
stf_task_add_dep(t, lData, STF_WRITE);
|
||||
stf_task_start(t);
|
||||
double* dData = (double*) stf_task_get(t, 0);
|
||||
fill_kernel<<<2, 128, 0, (cudaStream_t) stf_task_get_custream(t)>>>((int) N, dData, 42.0);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
|
||||
// Use host_launch to verify data on the host
|
||||
bool passed = false;
|
||||
verify_args vargs{N, &passed};
|
||||
|
||||
stf_host_launch_handle h = stf_host_launch_create(ctx);
|
||||
REQUIRE(h != nullptr);
|
||||
stf_host_launch_set_symbol(h, "verify");
|
||||
stf_host_launch_add_dep(h, lData, STF_READ);
|
||||
stf_host_launch_set_user_data(h, &vargs, sizeof(vargs), nullptr);
|
||||
stf_host_launch_submit(h, verify_callback);
|
||||
stf_host_launch_destroy(h);
|
||||
|
||||
stf_logical_data_destroy(lData);
|
||||
stf_ctx_finalize(ctx);
|
||||
|
||||
REQUIRE(passed);
|
||||
|
||||
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
|
||||
}
|
||||
|
||||
C2H_TEST("host_launch with graph context", "[host_launch]")
|
||||
{
|
||||
const size_t N = 1024;
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create_graph();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
double* host_data;
|
||||
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
host_data[i] = 0.0;
|
||||
}
|
||||
|
||||
stf_logical_data_handle lData = stf_logical_data(ctx, host_data, N * sizeof(double));
|
||||
REQUIRE(lData != nullptr);
|
||||
stf_logical_data_set_symbol(lData, "data");
|
||||
|
||||
// Fill data via a generic task with stream capture
|
||||
stf_task_handle t = stf_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_task_set_symbol(t, "fill");
|
||||
stf_task_add_dep(t, lData, STF_WRITE);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
double* dData = (double*) stf_task_get(t, 0);
|
||||
cudaStream_t stream = (cudaStream_t) stf_task_get_custream(t);
|
||||
fill_kernel<<<2, 128, 0, stream>>>((int) N, dData, 42.0);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
|
||||
// Use host_launch to verify data on the host
|
||||
bool passed = false;
|
||||
verify_args vargs{N, &passed};
|
||||
|
||||
stf_host_launch_handle h = stf_host_launch_create(ctx);
|
||||
REQUIRE(h != nullptr);
|
||||
stf_host_launch_set_symbol(h, "verify");
|
||||
stf_host_launch_add_dep(h, lData, STF_READ);
|
||||
stf_host_launch_set_user_data(h, &vargs, sizeof(vargs), nullptr);
|
||||
stf_host_launch_submit(h, verify_callback);
|
||||
stf_host_launch_destroy(h);
|
||||
|
||||
stf_logical_data_destroy(lData);
|
||||
stf_ctx_finalize(ctx);
|
||||
|
||||
REQUIRE(passed);
|
||||
|
||||
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
|
||||
}
|
||||
|
||||
C2H_TEST("host_launch with stackable context", "[host_launch][stackable]")
|
||||
{
|
||||
const size_t N = 1024;
|
||||
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
double* host_data;
|
||||
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
host_data[i] = 0.0;
|
||||
}
|
||||
|
||||
stf_logical_data_handle lData = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
|
||||
REQUIRE(lData != nullptr);
|
||||
stf_stackable_logical_data_set_symbol(lData, "data");
|
||||
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_task_set_symbol(t, "fill");
|
||||
stf_stackable_task_add_dep(ctx, t, lData, STF_WRITE);
|
||||
stf_task_start(t);
|
||||
double* dData = (double*) stf_task_get(t, 0);
|
||||
fill_kernel<<<2, 128, 0, (cudaStream_t) stf_task_get_custream(t)>>>((int) N, dData, 42.0);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
|
||||
bool passed = false;
|
||||
verify_args vargs{N, &passed};
|
||||
|
||||
stf_host_launch_handle h = stf_stackable_host_launch_create(ctx);
|
||||
REQUIRE(h != nullptr);
|
||||
stf_host_launch_set_symbol(h, "verify");
|
||||
stf_stackable_host_launch_add_dep(ctx, h, lData, STF_READ);
|
||||
stf_host_launch_set_user_data(h, &vargs, sizeof(vargs), nullptr);
|
||||
stf_stackable_host_launch_submit(h, verify_callback);
|
||||
stf_stackable_host_launch_destroy(h);
|
||||
|
||||
stf_stackable_logical_data_destroy(lData);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
|
||||
REQUIRE(passed);
|
||||
|
||||
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
|
||||
}
|
||||
|
||||
C2H_TEST("host_launch inside a stackable nested graph scope", "[host_launch][stackable]")
|
||||
{
|
||||
const size_t N = 1024;
|
||||
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
double* host_data;
|
||||
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
host_data[i] = 0.0;
|
||||
}
|
||||
|
||||
stf_logical_data_handle lData = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
|
||||
REQUIRE(lData != nullptr);
|
||||
|
||||
// Push a nested graph scope and run both the producer task and the host_launch
|
||||
// verifier inside it. The data auto-pushes from root to the nested scope.
|
||||
stf_stackable_push_graph(ctx);
|
||||
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, lData, STF_WRITE);
|
||||
// Inside a nested graph scope the task is captured into the child graph, so we
|
||||
// must enable capture to obtain the graph's capture stream. Otherwise
|
||||
// stf_task_get_custream() returns a null/uninitialized stream and the kernel
|
||||
// would run outside the STF graph, racing the host verifier below.
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
double* dData = (double*) stf_task_get(t, 0);
|
||||
fill_kernel<<<2, 128, 0, (cudaStream_t) stf_task_get_custream(t)>>>((int) N, dData, 42.0);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
|
||||
bool passed = false;
|
||||
verify_args vargs{N, &passed};
|
||||
|
||||
stf_host_launch_handle h = stf_stackable_host_launch_create(ctx);
|
||||
REQUIRE(h != nullptr);
|
||||
stf_stackable_host_launch_add_dep(ctx, h, lData, STF_READ);
|
||||
stf_host_launch_set_user_data(h, &vargs, sizeof(vargs), nullptr);
|
||||
stf_stackable_host_launch_submit(h, verify_callback);
|
||||
stf_stackable_host_launch_destroy(h);
|
||||
|
||||
stf_stackable_pop(ctx);
|
||||
|
||||
stf_stackable_logical_data_destroy(lData);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
|
||||
REQUIRE(passed);
|
||||
|
||||
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
|
||||
}
|
||||
37
cccl_upstream/c/experimental/stf/test/test_logical_data.cpp
Normal file
37
cccl_upstream/c/experimental/stf/test/test_logical_data.cpp
Normal file
@@ -0,0 +1,37 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <cccl/c/experimental/stf/stf.h>
|
||||
|
||||
C2H_TEST("basic stf logical_data", "[logical_data]")
|
||||
{
|
||||
size_t N = 1000000;
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
std::vector<float> A(N);
|
||||
std::vector<float> B(N);
|
||||
|
||||
stf_logical_data_handle lA = stf_logical_data(ctx, A.data(), N * sizeof(float));
|
||||
stf_logical_data_handle lB = stf_logical_data(ctx, B.data(), N * sizeof(float));
|
||||
REQUIRE(lA != nullptr);
|
||||
REQUIRE(lB != nullptr);
|
||||
|
||||
stf_logical_data_destroy(lA);
|
||||
stf_logical_data_destroy(lB);
|
||||
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Unit tests for stf_logical_data_with_place(): logical data with explicit
|
||||
// data place (host, pinned host, device).
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <cccl/c/experimental/stf/stf.h>
|
||||
|
||||
__global__ void scale_inplace(int n, float* data, float factor)
|
||||
{
|
||||
int i = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
|
||||
if (i < n)
|
||||
{
|
||||
data[i] *= factor;
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("stf_logical_data_with_place - host place (malloc)", "[logical_data_with_place]")
|
||||
{
|
||||
size_t N = 1024;
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
std::vector<float> A(N);
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
A[i] = static_cast<float>(i);
|
||||
}
|
||||
|
||||
stf_data_place_handle host_place = stf_data_place_host();
|
||||
stf_logical_data_handle lA = stf_logical_data_with_place(ctx, A.data(), N * sizeof(float), host_place);
|
||||
REQUIRE(lA != nullptr);
|
||||
stf_data_place_destroy(host_place);
|
||||
|
||||
stf_task_handle t = stf_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_task_add_dep(t, lA, STF_RW);
|
||||
stf_task_start(t);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
|
||||
stf_logical_data_destroy(lA);
|
||||
stf_ctx_finalize(ctx);
|
||||
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
REQUIRE(A[i] == static_cast<float>(i));
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("stf_logical_data_with_place - host place (pinned memory)", "[logical_data_with_place]")
|
||||
{
|
||||
size_t N = 1024;
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
float* A = nullptr;
|
||||
cudaError_t err = cudaMallocHost(&A, N * sizeof(float));
|
||||
REQUIRE(err == cudaSuccess);
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
A[i] = static_cast<float>(i);
|
||||
}
|
||||
|
||||
stf_data_place_handle host_place = stf_data_place_host();
|
||||
stf_logical_data_handle lA = stf_logical_data_with_place(ctx, A, N * sizeof(float), host_place);
|
||||
REQUIRE(lA != nullptr);
|
||||
stf_data_place_destroy(host_place);
|
||||
|
||||
stf_task_handle t = stf_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_task_add_dep(t, lA, STF_RW);
|
||||
stf_task_start(t);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
|
||||
stf_logical_data_destroy(lA);
|
||||
stf_ctx_finalize(ctx);
|
||||
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
REQUIRE(A[i] == static_cast<float>(i));
|
||||
}
|
||||
|
||||
REQUIRE(cudaFreeHost(A) == cudaSuccess);
|
||||
}
|
||||
|
||||
C2H_TEST("stf_logical_data_with_place - device place (data on current device)", "[logical_data_with_place]")
|
||||
{
|
||||
size_t N = 1024;
|
||||
const float factor = 2.0f;
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
float* d_raw = nullptr;
|
||||
cudaError_t err = cudaMalloc(&d_raw, N * sizeof(float));
|
||||
REQUIRE(err == cudaSuccess);
|
||||
std::unique_ptr<void, decltype(&cudaFree)> d_data_owner(d_raw, cudaFree);
|
||||
float* d_data = static_cast<float*>(d_data_owner.get());
|
||||
|
||||
std::vector<float> h_init(N);
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
h_init[i] = static_cast<float>(i);
|
||||
}
|
||||
err = cudaMemcpy(d_data, h_init.data(), N * sizeof(float), cudaMemcpyHostToDevice);
|
||||
REQUIRE(err == cudaSuccess);
|
||||
|
||||
stf_data_place_handle dev_place = stf_data_place_device(0);
|
||||
stf_logical_data_handle lD = stf_logical_data_with_place(ctx, d_data, N * sizeof(float), dev_place);
|
||||
REQUIRE(lD != nullptr);
|
||||
stf_data_place_destroy(dev_place);
|
||||
stf_logical_data_set_symbol(lD, "device_buf");
|
||||
|
||||
stf_cuda_kernel_handle k = stf_cuda_kernel_create(ctx);
|
||||
REQUIRE(k != nullptr);
|
||||
stf_cuda_kernel_set_symbol(k, "scale_inplace");
|
||||
stf_cuda_kernel_add_dep(k, lD, STF_RW);
|
||||
stf_cuda_kernel_start(k);
|
||||
float* arg_ptr = static_cast<float*>(stf_cuda_kernel_get_arg(k, 0));
|
||||
REQUIRE(arg_ptr == d_data);
|
||||
int n = static_cast<int>(N);
|
||||
const void* args[3] = {&n, &arg_ptr, &factor};
|
||||
dim3 grid(4);
|
||||
dim3 block(256);
|
||||
err = stf_cuda_kernel_add_desc(k, reinterpret_cast<void*>(scale_inplace), grid, block, 0, 3, args);
|
||||
REQUIRE(err == cudaSuccess);
|
||||
stf_cuda_kernel_end(k);
|
||||
stf_cuda_kernel_destroy(k);
|
||||
|
||||
stf_logical_data_destroy(lD);
|
||||
stf_ctx_finalize(ctx);
|
||||
|
||||
// Copy back and verify: should be i * factor
|
||||
std::vector<float> h_result(N);
|
||||
err = cudaMemcpy(h_result.data(), d_data, N * sizeof(float), cudaMemcpyDeviceToHost);
|
||||
REQUIRE(err == cudaSuccess);
|
||||
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
REQUIRE(h_result[i] == static_cast<float>(i) * factor);
|
||||
}
|
||||
}
|
||||
679
cccl_upstream/c/experimental/stf/test/test_places.cpp
Normal file
679
cccl_upstream/c/experimental/stf/test/test_places.cpp
Normal file
@@ -0,0 +1,679 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/__cmath/ceil_div.h>
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <cccl/c/experimental/stf/stf.h>
|
||||
|
||||
// Blocked partition along first dimension: maps data coordinates to grid position.
|
||||
// Used to exercise composite data place with a grid of execution places.
|
||||
static void blocked_mapper_1d(stf_pos4* result, stf_pos4 data_coords, stf_dim4 data_dims, stf_dim4 grid_dims)
|
||||
{
|
||||
uint64_t extent = data_dims.x;
|
||||
uint64_t nplaces = grid_dims.x;
|
||||
uint64_t part_size = ::cuda::ceil_div(extent, nplaces);
|
||||
if (part_size == 0)
|
||||
{
|
||||
part_size = 1;
|
||||
}
|
||||
int64_t c = static_cast<int64_t>(data_coords.x);
|
||||
int64_t place_x = c / static_cast<int64_t>(part_size);
|
||||
if (place_x >= static_cast<int64_t>(nplaces))
|
||||
{
|
||||
place_x = static_cast<int64_t>(nplaces) - 1;
|
||||
}
|
||||
result->x = place_x;
|
||||
result->y = 0;
|
||||
result->z = 0;
|
||||
result->t = 0;
|
||||
}
|
||||
|
||||
C2H_TEST("exec place from an externally-owned CUDA context", "[task][places][cuda_context]")
|
||||
{
|
||||
constexpr size_t element_count{1024};
|
||||
|
||||
// Wrap the primary context of device 0 as an exec place
|
||||
CUdevice dev = 0;
|
||||
REQUIRE(cuDeviceGet(&dev, 0) == CUDA_SUCCESS);
|
||||
CUcontext primary_ctx = nullptr;
|
||||
REQUIRE(cuDevicePrimaryCtxRetain(&primary_ctx, dev) == CUDA_SUCCESS);
|
||||
|
||||
// Null context is rejected
|
||||
REQUIRE(stf_exec_place_cuda_context(nullptr, 0) == nullptr);
|
||||
|
||||
// dev_id < 0 is derived from the context
|
||||
const stf_exec_place_handle place_derived = stf_exec_place_cuda_context(primary_ctx, -1);
|
||||
REQUIRE(place_derived != nullptr);
|
||||
REQUIRE(stf_exec_place_is_device(place_derived) != 0);
|
||||
stf_exec_place_destroy(place_derived);
|
||||
|
||||
const stf_exec_place_handle place = stf_exec_place_cuda_context(primary_ctx, 0);
|
||||
REQUIRE(place != nullptr);
|
||||
REQUIRE(stf_exec_place_is_device(place) != 0);
|
||||
REQUIRE(stf_exec_place_is_host(place) == 0);
|
||||
|
||||
// Run a task on the place and fill the buffer through its stream
|
||||
const stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
std::vector<float> x(element_count, 1.0f);
|
||||
const stf_logical_data_handle logical_x = stf_logical_data(ctx, x.data(), element_count * sizeof(float));
|
||||
REQUIRE(logical_x != nullptr);
|
||||
|
||||
const stf_task_handle task = stf_task_create(ctx);
|
||||
REQUIRE(task != nullptr);
|
||||
stf_task_set_exec_place(task, place);
|
||||
stf_task_add_dep(task, logical_x, STF_RW);
|
||||
stf_task_start(task);
|
||||
const CUstream stream = stf_task_get_custream(task);
|
||||
REQUIRE(stream != nullptr);
|
||||
float* const device_x = static_cast<float*>(stf_task_get(task, 0));
|
||||
REQUIRE(device_x != nullptr);
|
||||
REQUIRE(cudaMemsetAsync(device_x, 0, element_count * sizeof(float), stream) == cudaSuccess);
|
||||
stf_task_end(task);
|
||||
stf_task_destroy(task);
|
||||
|
||||
stf_logical_data_destroy(logical_x);
|
||||
stf_ctx_finalize(ctx);
|
||||
|
||||
for (size_t i = 0; i < element_count; i++)
|
||||
{
|
||||
REQUIRE(x[i] == 0.0f);
|
||||
}
|
||||
|
||||
stf_exec_place_destroy(place);
|
||||
REQUIRE(cuDevicePrimaryCtxRelease(dev) == CUDA_SUCCESS);
|
||||
}
|
||||
|
||||
C2H_TEST("empty stf tasks", "[task]")
|
||||
{
|
||||
size_t N = 1000000;
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
std::vector<float> X(N);
|
||||
std::vector<float> Y(N);
|
||||
std::vector<float> Z(N);
|
||||
|
||||
stf_logical_data_handle lX = stf_logical_data(ctx, X.data(), N * sizeof(float));
|
||||
stf_logical_data_handle lY = stf_logical_data(ctx, Y.data(), N * sizeof(float));
|
||||
stf_logical_data_handle lZ = stf_logical_data(ctx, Z.data(), N * sizeof(float));
|
||||
REQUIRE(lX != nullptr);
|
||||
REQUIRE(lY != nullptr);
|
||||
REQUIRE(lZ != nullptr);
|
||||
|
||||
stf_logical_data_set_symbol(lX, "X");
|
||||
stf_logical_data_set_symbol(lY, "Y");
|
||||
stf_logical_data_set_symbol(lZ, "Z");
|
||||
|
||||
stf_task_handle t1 = stf_task_create(ctx);
|
||||
REQUIRE(t1 != nullptr);
|
||||
stf_task_set_symbol(t1, "T1");
|
||||
stf_task_add_dep(t1, lX, STF_RW);
|
||||
stf_task_start(t1);
|
||||
stf_task_end(t1);
|
||||
stf_task_destroy(t1);
|
||||
|
||||
stf_task_handle t2 = stf_task_create(ctx);
|
||||
REQUIRE(t2 != nullptr);
|
||||
stf_task_set_symbol(t2, "T2");
|
||||
stf_task_add_dep(t2, lX, STF_READ);
|
||||
stf_task_add_dep(t2, lY, STF_RW);
|
||||
stf_task_start(t2);
|
||||
stf_task_end(t2);
|
||||
stf_task_destroy(t2);
|
||||
|
||||
stf_task_handle t3 = stf_task_create(ctx);
|
||||
REQUIRE(t3 != nullptr);
|
||||
stf_task_set_symbol(t3, "T3");
|
||||
stf_exec_place_handle e_place_dev0 = stf_exec_place_device(0);
|
||||
stf_task_set_exec_place(t3, e_place_dev0);
|
||||
stf_exec_place_destroy(e_place_dev0);
|
||||
stf_task_add_dep(t3, lX, STF_READ);
|
||||
stf_task_add_dep(t3, lZ, STF_RW);
|
||||
stf_task_start(t3);
|
||||
stf_task_end(t3);
|
||||
stf_task_destroy(t3);
|
||||
|
||||
stf_task_handle t4 = stf_task_create(ctx);
|
||||
REQUIRE(t4 != nullptr);
|
||||
stf_task_set_symbol(t4, "T4");
|
||||
stf_task_add_dep(t4, lY, STF_READ);
|
||||
stf_data_place_handle d_place_dev0 = stf_data_place_device(0);
|
||||
stf_task_add_dep_with_dplace(t4, lZ, STF_RW, d_place_dev0);
|
||||
stf_data_place_destroy(d_place_dev0);
|
||||
stf_task_start(t4);
|
||||
stf_task_end(t4);
|
||||
stf_task_destroy(t4);
|
||||
|
||||
stf_logical_data_destroy(lX);
|
||||
stf_logical_data_destroy(lY);
|
||||
stf_logical_data_destroy(lZ);
|
||||
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
C2H_TEST("composite data place with grid of places (same device repeated)", "[task][places][composite]")
|
||||
{
|
||||
const size_t nplaces = 3;
|
||||
stf_exec_place_handle places[3];
|
||||
for (auto& place : places)
|
||||
{
|
||||
place = stf_exec_place_device(0);
|
||||
}
|
||||
|
||||
stf_exec_place_handle grid = stf_exec_place_grid_create(places, nplaces, nullptr);
|
||||
REQUIRE(grid != nullptr);
|
||||
for (auto& place : places)
|
||||
{
|
||||
stf_exec_place_destroy(place);
|
||||
}
|
||||
|
||||
stf_data_place_handle composite_dplace = stf_data_place_composite(grid, blocked_mapper_1d);
|
||||
REQUIRE(composite_dplace != nullptr);
|
||||
stf_exec_place_grid_destroy(grid);
|
||||
|
||||
size_t N = 1024;
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
std::vector<float> X(N);
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
X[i] = static_cast<float>(i);
|
||||
}
|
||||
|
||||
stf_logical_data_handle lX = stf_logical_data(ctx, X.data(), N * sizeof(float));
|
||||
REQUIRE(lX != nullptr);
|
||||
stf_logical_data_set_symbol(lX, "X_composite");
|
||||
|
||||
stf_task_handle t = stf_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_task_set_symbol(t, "T_composite");
|
||||
stf_exec_place_handle e_place_dev0 = stf_exec_place_device(0);
|
||||
stf_task_set_exec_place(t, e_place_dev0);
|
||||
stf_exec_place_destroy(e_place_dev0);
|
||||
stf_task_add_dep_with_dplace(t, lX, STF_RW, composite_dplace);
|
||||
stf_task_start(t);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
|
||||
stf_data_place_destroy(composite_dplace);
|
||||
|
||||
stf_logical_data_destroy(lX);
|
||||
stf_ctx_finalize(ctx);
|
||||
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
REQUIRE(X[i] == static_cast<float>(i));
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("composite data place with stf_exec_place_grid_create (vector of places + dim4)", "[task][places][composite]")
|
||||
{
|
||||
const size_t nplaces = 4;
|
||||
stf_exec_place_handle places[4];
|
||||
for (auto& place : places)
|
||||
{
|
||||
place = stf_exec_place_device(0);
|
||||
}
|
||||
|
||||
stf_exec_place_handle grid_linear = stf_exec_place_grid_create(places, nplaces, nullptr);
|
||||
REQUIRE(grid_linear != nullptr);
|
||||
for (auto& place : places)
|
||||
{
|
||||
stf_exec_place_destroy(place);
|
||||
}
|
||||
stf_exec_place_grid_destroy(grid_linear);
|
||||
|
||||
for (auto& place : places)
|
||||
{
|
||||
place = stf_exec_place_device(0);
|
||||
}
|
||||
stf_dim4 grid_dims = {2, 2, 1, 1};
|
||||
stf_exec_place_handle grid = stf_exec_place_grid_create(places, nplaces, &grid_dims);
|
||||
REQUIRE(grid != nullptr);
|
||||
for (auto& place : places)
|
||||
{
|
||||
stf_exec_place_destroy(place);
|
||||
}
|
||||
|
||||
stf_data_place_handle composite_dplace = stf_data_place_composite(grid, blocked_mapper_1d);
|
||||
REQUIRE(composite_dplace != nullptr);
|
||||
stf_exec_place_grid_destroy(grid);
|
||||
|
||||
size_t N = 512;
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
std::vector<float> X(N);
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
X[i] = static_cast<float>(i);
|
||||
}
|
||||
|
||||
stf_logical_data_handle lX = stf_logical_data(ctx, X.data(), N * sizeof(float));
|
||||
REQUIRE(lX != nullptr);
|
||||
stf_task_handle t = stf_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_exec_place_handle e_place = stf_exec_place_device(0);
|
||||
stf_task_set_exec_place(t, e_place);
|
||||
stf_exec_place_destroy(e_place);
|
||||
stf_task_add_dep_with_dplace(t, lX, STF_RW, composite_dplace);
|
||||
stf_task_start(t);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
|
||||
stf_data_place_destroy(composite_dplace);
|
||||
|
||||
stf_logical_data_destroy(lX);
|
||||
stf_ctx_finalize(ctx);
|
||||
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
REQUIRE(X[i] == static_cast<float>(i));
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("task on exec_place_grid: get_grid_dims and get_custream_at_index", "[task][places][grid]")
|
||||
{
|
||||
const size_t nplaces = 2;
|
||||
stf_exec_place_handle places[2];
|
||||
for (auto& place : places)
|
||||
{
|
||||
place = stf_exec_place_device(0);
|
||||
}
|
||||
stf_exec_place_handle grid = stf_exec_place_grid_create(places, nplaces, nullptr);
|
||||
REQUIRE(grid != nullptr);
|
||||
for (auto& place : places)
|
||||
{
|
||||
stf_exec_place_destroy(place);
|
||||
}
|
||||
|
||||
stf_data_place_handle composite_dplace = stf_data_place_composite(grid, blocked_mapper_1d);
|
||||
REQUIRE(composite_dplace != nullptr);
|
||||
stf_exec_place_set_affine_data_place(grid, composite_dplace);
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
std::vector<float> X(4, 0.0f);
|
||||
|
||||
stf_logical_data_handle lX = stf_logical_data(ctx, X.data(), X.size() * sizeof(float));
|
||||
REQUIRE(lX != nullptr);
|
||||
|
||||
stf_task_handle t = stf_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_task_set_exec_place(t, grid);
|
||||
stf_task_add_dep(t, lX, STF_RW);
|
||||
stf_task_start(t);
|
||||
|
||||
stf_dim4 dims;
|
||||
int got_dims = stf_task_get_grid_dims(t, &dims);
|
||||
REQUIRE(got_dims == 0);
|
||||
REQUIRE(dims.x == 2);
|
||||
REQUIRE(dims.y == 1);
|
||||
REQUIRE(dims.z == 1);
|
||||
REQUIRE(dims.t == 1);
|
||||
|
||||
CUstream s0, s1;
|
||||
REQUIRE(stf_task_get_custream_at_index(t, 0, &s0) == 0);
|
||||
REQUIRE(stf_task_get_custream_at_index(t, 1, &s1) == 0);
|
||||
REQUIRE(s0 != nullptr);
|
||||
REQUIRE(s1 != nullptr);
|
||||
|
||||
// Out-of-range linear index must report an error rather than reading past the stream grid.
|
||||
CUstream s_oob;
|
||||
REQUIRE(stf_task_get_custream_at_index(t, 2, &s_oob) != 0);
|
||||
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
|
||||
stf_data_place_destroy(composite_dplace);
|
||||
stf_exec_place_grid_destroy(grid);
|
||||
stf_logical_data_destroy(lX);
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
C2H_TEST("task get_grid_dims returns error for non-grid exec_place", "[task][places][grid]")
|
||||
{
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
float val = 0.0f;
|
||||
auto lX = stf_logical_data(ctx, &val, sizeof(float));
|
||||
auto e_dev0 = stf_exec_place_device(0);
|
||||
|
||||
stf_task_handle t = stf_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_task_set_exec_place(t, e_dev0);
|
||||
stf_task_add_dep(t, lX, STF_RW);
|
||||
stf_task_start(t);
|
||||
|
||||
stf_dim4 dims;
|
||||
REQUIRE(stf_task_get_grid_dims(t, &dims) != 0);
|
||||
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
|
||||
stf_exec_place_destroy(e_dev0);
|
||||
stf_logical_data_destroy(lX);
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
// ===== Place scope and accessor tests (task-free usage) =====
|
||||
|
||||
C2H_TEST("exec_place_scope enter/exit", "[places][scope]")
|
||||
{
|
||||
stf_machine_init();
|
||||
stf_exec_place_handle dev0 = stf_exec_place_device(0);
|
||||
REQUIRE(dev0 != nullptr);
|
||||
|
||||
stf_exec_place_scope_handle scope = stf_exec_place_scope_enter(dev0, 0);
|
||||
REQUIRE(scope != nullptr);
|
||||
|
||||
stf_exec_place_scope_exit(scope);
|
||||
stf_exec_place_scope_exit(nullptr);
|
||||
|
||||
stf_exec_place_destroy(dev0);
|
||||
}
|
||||
|
||||
C2H_TEST("exec_place_scope nested", "[places][scope]")
|
||||
{
|
||||
stf_machine_init();
|
||||
stf_exec_place_handle dev0 = stf_exec_place_device(0);
|
||||
REQUIRE(dev0 != nullptr);
|
||||
|
||||
stf_exec_place_scope_handle outer = stf_exec_place_scope_enter(dev0, 0);
|
||||
REQUIRE(outer != nullptr);
|
||||
|
||||
stf_exec_place_scope_handle inner = stf_exec_place_scope_enter(dev0, 0);
|
||||
REQUIRE(inner != nullptr);
|
||||
|
||||
stf_exec_place_scope_exit(inner);
|
||||
stf_exec_place_scope_exit(outer);
|
||||
|
||||
stf_exec_place_destroy(dev0);
|
||||
}
|
||||
|
||||
C2H_TEST("exec_place_get_affine_data_place", "[places][accessor]")
|
||||
{
|
||||
stf_exec_place_handle dev0 = stf_exec_place_device(0);
|
||||
REQUIRE(dev0 != nullptr);
|
||||
|
||||
stf_data_place_handle dp = stf_exec_place_get_affine_data_place(dev0);
|
||||
REQUIRE(dp != nullptr);
|
||||
REQUIRE(stf_data_place_get_device_ordinal(dp) == 0);
|
||||
|
||||
stf_data_place_destroy(dp);
|
||||
stf_exec_place_destroy(dev0);
|
||||
}
|
||||
|
||||
C2H_TEST("exec_place_pick_stream standalone", "[places][scope][stream]")
|
||||
{
|
||||
stf_machine_init();
|
||||
// Standalone use: no STF context required, just a registry the caller owns.
|
||||
stf_exec_place_resources_handle res = stf_exec_place_resources_create();
|
||||
REQUIRE(res != nullptr);
|
||||
|
||||
stf_exec_place_handle dev0 = stf_exec_place_device(0);
|
||||
REQUIRE(dev0 != nullptr);
|
||||
|
||||
stf_exec_place_scope_handle scope = stf_exec_place_scope_enter(dev0, 0);
|
||||
REQUIRE(scope != nullptr);
|
||||
|
||||
CUstream s = stf_exec_place_pick_stream(res, dev0, /*for_computation=*/1);
|
||||
REQUIRE(s != nullptr);
|
||||
|
||||
stf_exec_place_scope_exit(scope);
|
||||
stf_exec_place_destroy(dev0);
|
||||
stf_exec_place_resources_destroy(res);
|
||||
}
|
||||
|
||||
C2H_TEST("exec_place resources are independent", "[places][scope][stream]")
|
||||
{
|
||||
stf_machine_init();
|
||||
stf_exec_place_resources_handle res1 = stf_exec_place_resources_create();
|
||||
stf_exec_place_resources_handle res2 = stf_exec_place_resources_create();
|
||||
REQUIRE(res1 != nullptr);
|
||||
REQUIRE(res2 != nullptr);
|
||||
|
||||
stf_exec_place_handle dev0 = stf_exec_place_device(0);
|
||||
REQUIRE(dev0 != nullptr);
|
||||
|
||||
stf_exec_place_scope_handle scope = stf_exec_place_scope_enter(dev0, 0);
|
||||
REQUIRE(scope != nullptr);
|
||||
|
||||
CUstream stream1 = stf_exec_place_pick_stream(res1, dev0, /*for_computation=*/1);
|
||||
CUstream stream2 = stf_exec_place_pick_stream(res2, dev0, /*for_computation=*/1);
|
||||
REQUIRE(stream1 != nullptr);
|
||||
REQUIRE(stream2 != nullptr);
|
||||
REQUIRE(stream1 != stream2);
|
||||
|
||||
stf_exec_place_scope_exit(scope);
|
||||
stf_exec_place_destroy(dev0);
|
||||
stf_exec_place_resources_destroy(res2);
|
||||
stf_exec_place_resources_destroy(res1);
|
||||
}
|
||||
|
||||
C2H_TEST("exec_place_pick_stream borrowed from context", "[places][scope][stream][ctx]")
|
||||
{
|
||||
stf_machine_init();
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
stf_exec_place_resources_handle res = stf_ctx_get_place_resources(ctx);
|
||||
REQUIRE(res != nullptr);
|
||||
|
||||
stf_exec_place_handle dev0 = stf_exec_place_device(0);
|
||||
stf_exec_place_scope_handle scope = stf_exec_place_scope_enter(dev0, 0);
|
||||
|
||||
CUstream s = stf_exec_place_pick_stream(res, dev0, /*for_computation=*/1);
|
||||
REQUIRE(s != nullptr);
|
||||
|
||||
stf_exec_place_scope_exit(scope);
|
||||
stf_exec_place_destroy(dev0);
|
||||
// `res` is a non-owning wrapper around context resources; destroy only the wrapper.
|
||||
stf_exec_place_resources_destroy(res);
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
C2H_TEST("exec_place_get_place on grid", "[places][accessor][grid]")
|
||||
{
|
||||
const size_t nplaces = 2;
|
||||
int device_ids[2] = {0, 0};
|
||||
stf_exec_place_handle grid = stf_exec_place_grid_from_devices(device_ids, nplaces);
|
||||
REQUIRE(grid != nullptr);
|
||||
|
||||
stf_exec_place_handle sub0 = stf_exec_place_get_place(grid, 0);
|
||||
stf_exec_place_handle sub1 = stf_exec_place_get_place(grid, 1);
|
||||
REQUIRE(sub0 != nullptr);
|
||||
REQUIRE(sub1 != nullptr);
|
||||
REQUIRE(stf_exec_place_is_device(sub0) != 0);
|
||||
REQUIRE(stf_exec_place_is_device(sub1) != 0);
|
||||
|
||||
stf_exec_place_destroy(sub0);
|
||||
stf_exec_place_destroy(sub1);
|
||||
stf_exec_place_grid_destroy(grid);
|
||||
}
|
||||
|
||||
C2H_TEST("exec_place_get_place on scalar", "[places][accessor]")
|
||||
{
|
||||
stf_exec_place_handle dev0 = stf_exec_place_device(0);
|
||||
REQUIRE(dev0 != nullptr);
|
||||
|
||||
stf_exec_place_handle sub = stf_exec_place_get_place(dev0, 0);
|
||||
REQUIRE(sub != nullptr);
|
||||
REQUIRE(stf_exec_place_is_device(sub) != 0);
|
||||
|
||||
stf_exec_place_destroy(sub);
|
||||
stf_exec_place_destroy(dev0);
|
||||
}
|
||||
|
||||
C2H_TEST("exec_place_get_place out of bounds", "[places][accessor]")
|
||||
{
|
||||
stf_exec_place_handle dev0 = stf_exec_place_device(0);
|
||||
REQUIRE(dev0 != nullptr);
|
||||
REQUIRE(stf_exec_place_get_place(dev0, 1) == nullptr);
|
||||
stf_exec_place_destroy(dev0);
|
||||
|
||||
int device_ids[2] = {0, 0};
|
||||
stf_exec_place_handle grid = stf_exec_place_grid_from_devices(device_ids, 2);
|
||||
REQUIRE(grid != nullptr);
|
||||
REQUIRE(stf_exec_place_get_place(grid, 2) == nullptr);
|
||||
stf_exec_place_grid_destroy(grid);
|
||||
}
|
||||
|
||||
C2H_TEST("machine_init idempotent", "[places][machine]")
|
||||
{
|
||||
stf_machine_init();
|
||||
stf_machine_init();
|
||||
}
|
||||
|
||||
C2H_TEST("green_context_helper and green-context places", "[places][green_ctx]")
|
||||
{
|
||||
#if !defined(CUDART_VERSION) || CUDART_VERSION < 12040
|
||||
REQUIRE(stf_green_context_helper_create(1, 0) == nullptr);
|
||||
#else
|
||||
stf_machine_init();
|
||||
stf_green_context_helper_handle helper = stf_green_context_helper_create(1, 0);
|
||||
if (helper == nullptr)
|
||||
{
|
||||
SKIP("green context support is not available");
|
||||
}
|
||||
|
||||
REQUIRE(stf_green_context_helper_get_device_id(helper) == 0);
|
||||
const size_t count = stf_green_context_helper_get_count(helper);
|
||||
REQUIRE(count >= 1);
|
||||
|
||||
stf_exec_place_handle default_affine_ep = stf_exec_place_green_ctx(helper, 0, /*use_green_ctx_data_place=*/0);
|
||||
REQUIRE(default_affine_ep != nullptr);
|
||||
REQUIRE(stf_exec_place_is_device(default_affine_ep) != 0);
|
||||
|
||||
stf_data_place_handle default_affine_dp = stf_exec_place_get_affine_data_place(default_affine_ep);
|
||||
REQUIRE(default_affine_dp != nullptr);
|
||||
REQUIRE(stf_data_place_get_device_ordinal(default_affine_dp) == 0);
|
||||
|
||||
stf_exec_place_handle green_affine_ep = stf_exec_place_green_ctx(helper, 0, /*use_green_ctx_data_place=*/1);
|
||||
REQUIRE(green_affine_ep != nullptr);
|
||||
REQUIRE(stf_exec_place_is_device(green_affine_ep) != 0);
|
||||
|
||||
stf_data_place_handle green_affine_dp = stf_exec_place_get_affine_data_place(green_affine_ep);
|
||||
REQUIRE(green_affine_dp != nullptr);
|
||||
REQUIRE(stf_data_place_get_device_ordinal(green_affine_dp) == 0);
|
||||
const std::string green_affine_desc = stf_data_place_to_string(green_affine_dp);
|
||||
REQUIRE(green_affine_desc.find("green_ctx") != std::string::npos);
|
||||
|
||||
stf_data_place_handle green_dp = stf_data_place_green_ctx(helper, 0);
|
||||
REQUIRE(green_dp != nullptr);
|
||||
REQUIRE(stf_data_place_get_device_ordinal(green_dp) == 0);
|
||||
REQUIRE(stf_data_place_allocation_is_stream_ordered(green_dp) == 1);
|
||||
const std::string green_dp_desc = stf_data_place_to_string(green_dp);
|
||||
REQUIRE(green_dp_desc.find("green_ctx") != std::string::npos);
|
||||
|
||||
REQUIRE(stf_exec_place_green_ctx(helper, count, /*use_green_ctx_data_place=*/0) == nullptr);
|
||||
REQUIRE(stf_data_place_green_ctx(helper, count) == nullptr);
|
||||
|
||||
stf_data_place_destroy(green_dp);
|
||||
stf_data_place_destroy(green_affine_dp);
|
||||
stf_exec_place_destroy(green_affine_ep);
|
||||
stf_data_place_destroy(default_affine_dp);
|
||||
stf_exec_place_destroy(default_affine_ep);
|
||||
stf_green_context_helper_destroy(helper);
|
||||
#endif
|
||||
}
|
||||
|
||||
C2H_TEST("data_place_allocate_device", "[places][allocate]")
|
||||
{
|
||||
stf_exec_place_resources_handle res = stf_exec_place_resources_create();
|
||||
stf_exec_place_handle ep = stf_exec_place_device(0);
|
||||
REQUIRE(ep != nullptr);
|
||||
|
||||
stf_exec_place_scope_handle scope = stf_exec_place_scope_enter(ep, 0);
|
||||
REQUIRE(scope != nullptr);
|
||||
|
||||
CUstream stream = stf_exec_place_pick_stream(res, ep, /*for_computation=*/0);
|
||||
stf_data_place_handle dplace = stf_exec_place_get_affine_data_place(ep);
|
||||
REQUIRE(dplace != nullptr);
|
||||
|
||||
void* ptr = stf_data_place_allocate(dplace, 1024, reinterpret_cast<cudaStream_t>(stream));
|
||||
REQUIRE(ptr != nullptr);
|
||||
|
||||
stf_data_place_deallocate(dplace, ptr, 1024, reinterpret_cast<cudaStream_t>(stream));
|
||||
|
||||
stf_data_place_destroy(dplace);
|
||||
stf_exec_place_scope_exit(scope);
|
||||
stf_exec_place_destroy(ep);
|
||||
stf_exec_place_resources_destroy(res);
|
||||
}
|
||||
|
||||
C2H_TEST("data_place_allocate_host", "[places][allocate]")
|
||||
{
|
||||
stf_data_place_handle dplace = stf_data_place_host();
|
||||
REQUIRE(dplace != nullptr);
|
||||
|
||||
void* ptr = stf_data_place_allocate(dplace, 256, nullptr);
|
||||
REQUIRE(ptr != nullptr);
|
||||
|
||||
int* buf = static_cast<int*>(ptr);
|
||||
buf[0] = 42;
|
||||
REQUIRE(buf[0] == 42);
|
||||
|
||||
stf_data_place_deallocate(dplace, ptr, 256, nullptr);
|
||||
stf_data_place_destroy(dplace);
|
||||
}
|
||||
|
||||
C2H_TEST("data_place_allocate_managed", "[places][allocate]")
|
||||
{
|
||||
stf_data_place_handle dplace = stf_data_place_managed();
|
||||
REQUIRE(dplace != nullptr);
|
||||
|
||||
void* ptr = stf_data_place_allocate(dplace, 512, nullptr);
|
||||
REQUIRE(ptr != nullptr);
|
||||
|
||||
int* buf = static_cast<int*>(ptr);
|
||||
buf[0] = 99;
|
||||
REQUIRE(buf[0] == 99);
|
||||
|
||||
stf_data_place_deallocate(dplace, ptr, 512, nullptr);
|
||||
stf_data_place_destroy(dplace);
|
||||
}
|
||||
|
||||
C2H_TEST("data_place_allocation_is_stream_ordered", "[places][allocate]")
|
||||
{
|
||||
stf_data_place_handle dev = stf_data_place_device(0);
|
||||
REQUIRE(dev != nullptr);
|
||||
REQUIRE(stf_data_place_allocation_is_stream_ordered(dev) == 1);
|
||||
stf_data_place_destroy(dev);
|
||||
|
||||
stf_data_place_handle host = stf_data_place_host();
|
||||
REQUIRE(host != nullptr);
|
||||
REQUIRE(stf_data_place_allocation_is_stream_ordered(host) == 0);
|
||||
stf_data_place_destroy(host);
|
||||
|
||||
stf_data_place_handle mgd = stf_data_place_managed();
|
||||
REQUIRE(mgd != nullptr);
|
||||
REQUIRE(stf_data_place_allocation_is_stream_ordered(mgd) == 0);
|
||||
stf_data_place_destroy(mgd);
|
||||
}
|
||||
|
||||
C2H_TEST("data_place_allocate_invalid_returns_null", "[places][allocate]")
|
||||
{
|
||||
stf_data_place_handle inv = stf_data_place_affine();
|
||||
REQUIRE(inv != nullptr);
|
||||
void* ptr = stf_data_place_allocate(inv, 64, nullptr);
|
||||
REQUIRE(ptr == nullptr);
|
||||
stf_data_place_destroy(inv);
|
||||
}
|
||||
761
cccl_upstream/c/experimental/stf/test/test_stackable.cu
Normal file
761
cccl_upstream/c/experimental/stf/test/test_stackable.cu
Normal file
@@ -0,0 +1,761 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <cccl/c/experimental/stf/stf.h>
|
||||
|
||||
__global__ void scale_kernel(int cnt, double* data, double factor)
|
||||
{
|
||||
const int tid = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
|
||||
const int nthreads = static_cast<int>(gridDim.x * blockDim.x);
|
||||
for (int i = tid; i < cnt; i += nthreads)
|
||||
{
|
||||
data[i] *= factor;
|
||||
}
|
||||
}
|
||||
|
||||
__global__ void increment_kernel(int cnt, double* data)
|
||||
{
|
||||
const int tid = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
|
||||
const int nthreads = static_cast<int>(gridDim.x * blockDim.x);
|
||||
for (int i = tid; i < cnt; i += nthreads)
|
||||
{
|
||||
data[i] += 1.0;
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("stackable: push_graph / pop", "[stackable]")
|
||||
{
|
||||
const size_t N = 256;
|
||||
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
double* host_data;
|
||||
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
host_data[i] = static_cast<double>(i);
|
||||
}
|
||||
|
||||
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
|
||||
REQUIRE(lA != nullptr);
|
||||
|
||||
// Multiply by 2 inside a nested graph scope.
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
double* d = static_cast<double*>(stf_task_get(t, 0));
|
||||
scale_kernel<<<2, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d, 2.0);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
stf_stackable_pop(ctx);
|
||||
|
||||
stf_stackable_logical_data_destroy(lA);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
REQUIRE(std::fabs(host_data[i] - 2.0 * static_cast<double>(i)) < 1e-10);
|
||||
}
|
||||
|
||||
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
|
||||
}
|
||||
|
||||
C2H_TEST("stackable: pop_prologue relaunch accumulates N times", "[stackable][launchable]")
|
||||
{
|
||||
const size_t N = 256;
|
||||
const int relaunchN = 16;
|
||||
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
double* host_data;
|
||||
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
host_data[i] = 0.0;
|
||||
}
|
||||
|
||||
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
|
||||
REQUIRE(lA != nullptr);
|
||||
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
double* d = static_cast<double*>(stf_task_get(t, 0));
|
||||
increment_kernel<<<2, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
|
||||
// Two-phase pop: instantiate the graph, launch it relaunchN times, then
|
||||
// run the epilogue to release resources and unfreeze lA.
|
||||
stf_launchable_graph_handle lh = stf_stackable_pop_prologue(ctx);
|
||||
REQUIRE(lh != nullptr);
|
||||
for (int k = 0; k < relaunchN; ++k)
|
||||
{
|
||||
stf_launchable_graph_launch(lh);
|
||||
}
|
||||
stf_stackable_pop_epilogue(ctx);
|
||||
stf_launchable_graph_destroy(lh);
|
||||
|
||||
stf_stackable_logical_data_destroy(lA);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
REQUIRE(std::fabs(host_data[i] - static_cast<double>(relaunchN)) < 1e-10);
|
||||
}
|
||||
|
||||
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
|
||||
}
|
||||
|
||||
C2H_TEST("stackable: pop_prologue with zero launches unfreezes", "[stackable][launchable]")
|
||||
{
|
||||
const size_t N = 128;
|
||||
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
double* host_data;
|
||||
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
host_data[i] = 7.0;
|
||||
}
|
||||
|
||||
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
|
||||
REQUIRE(lA != nullptr);
|
||||
|
||||
// Push + submit work, but never launch the graph. The epilogue must still
|
||||
// release resources so that lA is unfrozen and reusable below.
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
double* d = static_cast<double*>(stf_task_get(t, 0));
|
||||
increment_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
stf_launchable_graph_handle lh = stf_stackable_pop_prologue(ctx);
|
||||
REQUIRE(lh != nullptr);
|
||||
stf_stackable_pop_epilogue(ctx);
|
||||
stf_launchable_graph_destroy(lh);
|
||||
|
||||
// Normal push_graph/pop still works after a zero-launch prologue+epilogue.
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
double* d = static_cast<double*>(stf_task_get(t, 0));
|
||||
scale_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d, 2.0);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
stf_stackable_pop(ctx);
|
||||
|
||||
stf_stackable_logical_data_destroy(lA);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
|
||||
// Zero-launch means the first graph never ran. The second scope doubled
|
||||
// the initial 7.0 to 14.0.
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
REQUIRE(std::fabs(host_data[i] - 14.0) < 1e-10);
|
||||
}
|
||||
|
||||
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
|
||||
}
|
||||
|
||||
C2H_TEST("stackable: launchable exec and stream accessors are non-null", "[stackable][launchable]")
|
||||
{
|
||||
const size_t N = 64;
|
||||
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
double* host_data;
|
||||
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
host_data[i] = 0.0;
|
||||
}
|
||||
|
||||
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
|
||||
REQUIRE(lA != nullptr);
|
||||
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
double* d = static_cast<double*>(stf_task_get(t, 0));
|
||||
increment_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
stf_launchable_graph_handle lh = stf_stackable_pop_prologue(ctx);
|
||||
REQUIRE(lh != nullptr);
|
||||
|
||||
// Accessors must be valid between prologue and epilogue. graph() must
|
||||
// return a live cudaGraph_t without forcing instantiation, exec() must
|
||||
// return a live cudaGraphExec_t, stream() is pure observation.
|
||||
REQUIRE(stf_launchable_graph_graph(lh) != nullptr);
|
||||
REQUIRE(stf_launchable_graph_exec(lh) != nullptr);
|
||||
REQUIRE(stf_launchable_graph_stream(lh) != nullptr);
|
||||
|
||||
stf_launchable_graph_launch(lh);
|
||||
|
||||
stf_stackable_pop_epilogue(ctx);
|
||||
stf_launchable_graph_destroy(lh);
|
||||
|
||||
stf_stackable_logical_data_destroy(lA);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
REQUIRE(std::fabs(host_data[i] - 1.0) < 1e-10);
|
||||
}
|
||||
|
||||
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
|
||||
}
|
||||
|
||||
C2H_TEST("stackable: launchable graph() embed into outer graph", "[stackable][launchable]")
|
||||
{
|
||||
const size_t N = 64;
|
||||
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
double* host_data;
|
||||
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
host_data[i] = 0.0;
|
||||
}
|
||||
|
||||
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
|
||||
REQUIRE(lA != nullptr);
|
||||
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
double* d = static_cast<double*>(stf_task_get(t, 0));
|
||||
increment_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
stf_launchable_graph_handle lh = stf_stackable_pop_prologue(ctx);
|
||||
REQUIRE(lh != nullptr);
|
||||
|
||||
// Grab the underlying cudaGraph_t WITHOUT forcing instantiation and
|
||||
// without ever calling stf_launchable_graph_exec(). The child graph
|
||||
// built by the nested scope is embedded into an outer graph which is
|
||||
// instantiated and launched manually here.
|
||||
cudaGraph_t child_graph = stf_launchable_graph_graph(lh);
|
||||
REQUIRE(child_graph != nullptr);
|
||||
|
||||
cudaStream_t support_stream = stf_launchable_graph_stream(lh);
|
||||
REQUIRE(support_stream != nullptr);
|
||||
|
||||
cudaGraph_t outer = nullptr;
|
||||
REQUIRE(cudaGraphCreate(&outer, 0) == cudaSuccess);
|
||||
|
||||
cudaGraphNode_t child_node = nullptr;
|
||||
REQUIRE(cudaGraphAddChildGraphNode(&child_node, outer, nullptr, 0, child_graph) == cudaSuccess);
|
||||
|
||||
cudaGraphExec_t outer_exec = nullptr;
|
||||
#if _CCCL_CTK_AT_LEAST(12, 0)
|
||||
REQUIRE(cudaGraphInstantiate(&outer_exec, outer, 0) == cudaSuccess);
|
||||
#else
|
||||
REQUIRE(cudaGraphInstantiate(&outer_exec, outer, nullptr, nullptr, 0) == cudaSuccess);
|
||||
#endif
|
||||
|
||||
// Route the outer launch through the support stream: since graph() has
|
||||
// triggered the lazy dep-A sync on that stream, it is safe to drive
|
||||
// cudaGraphLaunch on it here.
|
||||
REQUIRE(cudaGraphLaunch(outer_exec, support_stream) == cudaSuccess);
|
||||
|
||||
REQUIRE(cudaGraphExecDestroy(outer_exec) == cudaSuccess);
|
||||
REQUIRE(cudaGraphDestroy(outer) == cudaSuccess);
|
||||
|
||||
stf_stackable_pop_epilogue(ctx);
|
||||
stf_launchable_graph_destroy(lh);
|
||||
|
||||
stf_stackable_logical_data_destroy(lA);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
REQUIRE(std::fabs(host_data[i] - 1.0) < 1e-10);
|
||||
}
|
||||
|
||||
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
|
||||
}
|
||||
|
||||
C2H_TEST("stackable: shared pop_prologue dup/free releases only at last free", "[stackable][launchable]")
|
||||
{
|
||||
const size_t N = 128;
|
||||
const int relaunchN = 5;
|
||||
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
double* host_data;
|
||||
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
host_data[i] = 0.0;
|
||||
}
|
||||
|
||||
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
|
||||
REQUIRE(lA != nullptr);
|
||||
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
double* d = static_cast<double*>(stf_task_get(t, 0));
|
||||
increment_kernel<<<2, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
|
||||
stf_launchable_graph_shared h1 = nullptr;
|
||||
REQUIRE(stf_stackable_pop_prologue_shared(ctx, &h1) == 0);
|
||||
REQUIRE(h1 != nullptr);
|
||||
REQUIRE(stf_launchable_graph_shared_valid(h1) == 1);
|
||||
REQUIRE(stf_launchable_graph_shared_stream(h1) != nullptr);
|
||||
|
||||
// Dup before launching anything: both handles must be able to drive the
|
||||
// same underlying graph.
|
||||
stf_launchable_graph_shared h2 = nullptr;
|
||||
REQUIRE(stf_launchable_graph_shared_dup(h1, &h2) == 0);
|
||||
REQUIRE(h2 != nullptr);
|
||||
REQUIRE(stf_launchable_graph_shared_valid(h2) == 1);
|
||||
|
||||
for (int k = 0; k < relaunchN; ++k)
|
||||
{
|
||||
// Alternate between the two handles - both must work.
|
||||
if ((k & 1) == 0)
|
||||
{
|
||||
stf_launchable_graph_shared_launch(h1);
|
||||
}
|
||||
else
|
||||
{
|
||||
stf_launchable_graph_shared_launch(h2);
|
||||
}
|
||||
}
|
||||
|
||||
// Free one handle; the other must still launch. No pop_epilogue yet.
|
||||
stf_launchable_graph_shared_free(h1);
|
||||
REQUIRE(stf_launchable_graph_shared_valid(h2) == 1);
|
||||
stf_launchable_graph_shared_launch(h2);
|
||||
|
||||
// Free the last handle: pop_epilogue runs automatically here.
|
||||
stf_launchable_graph_shared_free(h2);
|
||||
|
||||
// The context must be usable again after the shared release.
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
double* d = static_cast<double*>(stf_task_get(t, 0));
|
||||
scale_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d, 2.0);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
stf_stackable_pop(ctx);
|
||||
|
||||
stf_stackable_logical_data_destroy(lA);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
|
||||
// Each launch added +1; final scale doubled; relaunchN launches via h1/h2
|
||||
// plus one extra launch via h2 after free(h1) -> (relaunchN + 1) * 2.
|
||||
const double expected = 2.0 * (static_cast<double>(relaunchN) + 1.0);
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
REQUIRE(std::fabs(host_data[i] - expected) < 1e-10);
|
||||
}
|
||||
|
||||
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
|
||||
}
|
||||
|
||||
C2H_TEST("stackable: shared pop_prologue tolerates NULL free", "[stackable][launchable]")
|
||||
{
|
||||
// stf_launchable_graph_shared_free(NULL) must be a no-op just like the
|
||||
// other destroy entry points. The valid() probe returns 0 for NULL.
|
||||
stf_launchable_graph_shared_free(nullptr);
|
||||
REQUIRE(stf_launchable_graph_shared_valid(nullptr) == 0);
|
||||
}
|
||||
|
||||
C2H_TEST("stackable: nested push_graph scopes", "[stackable]")
|
||||
{
|
||||
const size_t N = 128;
|
||||
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
double* host_data;
|
||||
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
host_data[i] = 0.0;
|
||||
}
|
||||
|
||||
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
|
||||
REQUIRE(lA != nullptr);
|
||||
|
||||
// Two nested scopes: each scope adds 1.0, so after popping both we expect 2.0.
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
double* d = static_cast<double*>(stf_task_get(t, 0));
|
||||
increment_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
stf_task_handle t2 = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t2 != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t2, lA, STF_RW);
|
||||
stf_task_enable_capture(t2);
|
||||
stf_task_start(t2);
|
||||
double* d2 = static_cast<double*>(stf_task_get(t2, 0));
|
||||
increment_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t2)>>>(static_cast<int>(N), d2);
|
||||
stf_task_end(t2);
|
||||
stf_task_destroy(t2);
|
||||
}
|
||||
stf_stackable_pop(ctx);
|
||||
}
|
||||
stf_stackable_pop(ctx);
|
||||
|
||||
stf_stackable_logical_data_destroy(lA);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
REQUIRE(std::fabs(host_data[i] - 2.0) < 1e-10);
|
||||
}
|
||||
|
||||
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
|
||||
}
|
||||
|
||||
C2H_TEST("stackable: token + fence", "[stackable]")
|
||||
{
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
stf_logical_data_handle tok = stf_stackable_token(ctx);
|
||||
REQUIRE(tok != nullptr);
|
||||
|
||||
// Sequential task chain through the token: t1 (write) -> t2 (read).
|
||||
stf_task_handle t1 = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t1 != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t1, tok, STF_WRITE);
|
||||
stf_task_start(t1);
|
||||
stf_task_end(t1);
|
||||
stf_task_destroy(t1);
|
||||
|
||||
stf_task_handle t2 = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t2 != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t2, tok, STF_READ);
|
||||
stf_task_start(t2);
|
||||
stf_task_end(t2);
|
||||
stf_task_destroy(t2);
|
||||
|
||||
cudaStream_t fence = stf_stackable_ctx_fence(ctx);
|
||||
REQUIRE(cudaStreamSynchronize(fence) == cudaSuccess);
|
||||
|
||||
stf_stackable_token_destroy(tok);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
#if _CCCL_CTK_AT_LEAST(12, 4)
|
||||
|
||||
// Smoke test for the while/repeat C-API surface: create+destroy each kind of
|
||||
// scope without populating a body. Body-level integration is exercised at the
|
||||
// C++ level by cudax/test/stf/local_stf/stackable_nested_repeat.cu and the
|
||||
// graph_scope_test, but the C-API task-driven body still needs a follow-up to
|
||||
// nail down the right capture path; tracked separately.
|
||||
C2H_TEST("stackable: push_repeat / pop_repeat smoke", "[stackable][repeat]")
|
||||
{
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
stf_repeat_scope_handle scope = stf_stackable_push_repeat(ctx, /*count=*/1);
|
||||
REQUIRE(scope != nullptr);
|
||||
stf_stackable_pop_repeat(scope);
|
||||
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
C2H_TEST("stackable: push_while / pop_while smoke", "[stackable][while]")
|
||||
{
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
stf_while_scope_handle scope = stf_stackable_push_while(ctx);
|
||||
REQUIRE(scope != nullptr);
|
||||
|
||||
// The conditional handle is observable as a uint64_t; just sanity-check it.
|
||||
REQUIRE(stf_while_scope_get_cond_handle(scope) != 0);
|
||||
|
||||
stf_stackable_pop_while(scope);
|
||||
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
// Regression test mirroring probe_k_sweep.py: inside a while-scope body, chain
|
||||
// K tasks that each do .rw() on the same persistent logical data, and make the
|
||||
// loop execute exactly once. Sweep K=1..16 and expect every element of the
|
||||
// accumulator to equal K. The equivalent Python probe fails deterministically
|
||||
// when K is a multiple of 4 (drops exactly one update), so this test pins down
|
||||
// whether the bug is in the C-API task path or somewhere above it.
|
||||
C2H_TEST("stackable: while-body K chained rw tasks sweep", "[stackable][while][c-api]")
|
||||
{
|
||||
const int Nd = 128;
|
||||
const double tol_eps = 1e-10;
|
||||
int total_mismatches = 0;
|
||||
double total_off_by_one = 0.0;
|
||||
|
||||
for (int K = 1; K <= 16; ++K)
|
||||
{
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
// Accumulator: zero-initialized double[Nd].
|
||||
double* host_acc;
|
||||
REQUIRE(cudaMallocHost(&host_acc, Nd * sizeof(double)) == cudaSuccess);
|
||||
for (int i = 0; i < Nd; i++)
|
||||
{
|
||||
host_acc[i] = 0.0;
|
||||
}
|
||||
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_acc, Nd * sizeof(double));
|
||||
REQUIRE(lA != nullptr);
|
||||
|
||||
// "done" flag: starts at 1.0, body drives it to 0.0 so while stops after 1
|
||||
// iteration. We use a double scalar to keep it consistent with the kernel
|
||||
// family used by the probe.
|
||||
double* host_done;
|
||||
REQUIRE(cudaMallocHost(&host_done, sizeof(double)) == cudaSuccess);
|
||||
host_done[0] = 1.0;
|
||||
stf_logical_data_handle lD = stf_stackable_logical_data(ctx, host_done, sizeof(double));
|
||||
REQUIRE(lD != nullptr);
|
||||
|
||||
stf_while_scope_handle scope = stf_stackable_push_while(ctx);
|
||||
REQUIRE(scope != nullptr);
|
||||
{
|
||||
// K chained increments on lA, using the C-API raw task path that the
|
||||
// Python binding also uses.
|
||||
for (int k = 0; k < K; ++k)
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
double* d = static_cast<double*>(stf_task_get(t, 0));
|
||||
increment_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(Nd, d);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
|
||||
// Drive the done flag to 0.0 so the loop stops after 1 iteration.
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, lD, STF_WRITE);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
double* d = static_cast<double*>(stf_task_get(t, 0));
|
||||
scale_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>(1, d, 0.0);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
|
||||
// Continue while done > 0.5 (i.e. stop after we've zeroed it).
|
||||
stf_stackable_while_cond_scalar(ctx, scope, lD, STF_CMP_GT, 0.5, STF_DTYPE_FLOAT64);
|
||||
}
|
||||
stf_stackable_pop_while(scope);
|
||||
|
||||
stf_stackable_logical_data_destroy(lA);
|
||||
stf_stackable_logical_data_destroy(lD);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
|
||||
const double expected = static_cast<double>(K);
|
||||
int mismatches = 0;
|
||||
for (int i = 0; i < Nd; i++)
|
||||
{
|
||||
if (std::fabs(host_acc[i] - expected) > tol_eps)
|
||||
{
|
||||
++mismatches;
|
||||
}
|
||||
}
|
||||
if (mismatches != 0)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"[C-API K=%d] host_acc[0]=%g expected=%g (%d/%d mismatches)\n",
|
||||
K,
|
||||
host_acc[0],
|
||||
expected,
|
||||
mismatches,
|
||||
Nd);
|
||||
total_mismatches += mismatches;
|
||||
total_off_by_one += host_acc[0] - expected;
|
||||
}
|
||||
|
||||
REQUIRE(cudaFreeHost(host_acc) == cudaSuccess);
|
||||
REQUIRE(cudaFreeHost(host_done) == cudaSuccess);
|
||||
}
|
||||
|
||||
REQUIRE(total_mismatches == 0);
|
||||
(void) total_off_by_one;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
// Run a while loop whose body increments a 1-element double counter once per
|
||||
// iteration and leaves a 1-element flag at its initial value 1.0. The
|
||||
// continuation condition is built by `set_condition` from the counter and
|
||||
// flag handles. Returns the final counter value observed on the host.
|
||||
template <typename SetCondition>
|
||||
double run_compound_while(SetCondition&& set_condition)
|
||||
{
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
double* host_iter;
|
||||
REQUIRE(cudaMallocHost(&host_iter, sizeof(double)) == cudaSuccess);
|
||||
host_iter[0] = 0.0;
|
||||
double* host_flag;
|
||||
REQUIRE(cudaMallocHost(&host_flag, sizeof(double)) == cudaSuccess);
|
||||
host_flag[0] = 1.0;
|
||||
|
||||
stf_logical_data_handle lIter = stf_stackable_logical_data(ctx, host_iter, sizeof(double));
|
||||
REQUIRE(lIter != nullptr);
|
||||
stf_logical_data_handle lFlag = stf_stackable_logical_data(ctx, host_flag, sizeof(double));
|
||||
REQUIRE(lFlag != nullptr);
|
||||
|
||||
stf_while_scope_handle scope = stf_stackable_push_while(ctx);
|
||||
REQUIRE(scope != nullptr);
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, lIter, STF_RW);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
double* d = static_cast<double*>(stf_task_get(t, 0));
|
||||
increment_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>(1, d);
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
|
||||
set_condition(ctx, scope, lIter, lFlag);
|
||||
}
|
||||
stf_stackable_pop_while(scope);
|
||||
|
||||
stf_stackable_logical_data_destroy(lIter);
|
||||
stf_stackable_logical_data_destroy(lFlag);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
|
||||
const double result = host_iter[0];
|
||||
REQUIRE(cudaFreeHost(host_iter) == cudaSuccess);
|
||||
REQUIRE(cudaFreeHost(host_flag) == cudaSuccess);
|
||||
return result;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("stackable: while compound condition", "[stackable][while][c-api]")
|
||||
{
|
||||
SECTION("ALL combiner stops at the iteration cap")
|
||||
{
|
||||
// flag > 0.5 is always true; iter < 5 caps the loop at 5 iterations.
|
||||
const double iters = run_compound_while(
|
||||
[](stf_ctx_handle ctx, stf_while_scope_handle scope, stf_logical_data_handle lIter, stf_logical_data_handle lFlag) {
|
||||
stf_while_cond_term terms[2] = {
|
||||
{lFlag, STF_CMP_GT, 0.5, STF_DTYPE_FLOAT64, 0},
|
||||
{lIter, STF_CMP_LT, 5.0, STF_DTYPE_FLOAT64, 0},
|
||||
};
|
||||
stf_stackable_while_cond_multi(ctx, scope, terms, 2, STF_COND_ALL);
|
||||
});
|
||||
REQUIRE(iters == 5.0);
|
||||
}
|
||||
|
||||
SECTION("ANY combiner with a negated term")
|
||||
{
|
||||
// ~(flag > 0.5) is always false, so only iter < 3 keeps the loop going.
|
||||
const double iters = run_compound_while(
|
||||
[](stf_ctx_handle ctx, stf_while_scope_handle scope, stf_logical_data_handle lIter, stf_logical_data_handle lFlag) {
|
||||
stf_while_cond_term terms[2] = {
|
||||
{lIter, STF_CMP_LT, 3.0, STF_DTYPE_FLOAT64, 0},
|
||||
{lFlag, STF_CMP_GT, 0.5, STF_DTYPE_FLOAT64, 1},
|
||||
};
|
||||
stf_stackable_while_cond_multi(ctx, scope, terms, 2, STF_COND_ANY);
|
||||
});
|
||||
REQUIRE(iters == 3.0);
|
||||
}
|
||||
|
||||
SECTION("duplicate logical data across terms shares one dependency")
|
||||
{
|
||||
const double iters = run_compound_while(
|
||||
[](stf_ctx_handle ctx,
|
||||
stf_while_scope_handle scope,
|
||||
stf_logical_data_handle lIter,
|
||||
stf_logical_data_handle /*lFlag*/) {
|
||||
stf_while_cond_term terms[2] = {
|
||||
{lIter, STF_CMP_LT, 4.0, STF_DTYPE_FLOAT64, 0},
|
||||
{lIter, STF_CMP_GT, -1.0, STF_DTYPE_FLOAT64, 0},
|
||||
};
|
||||
stf_stackable_while_cond_multi(ctx, scope, terms, 2, STF_COND_ALL);
|
||||
});
|
||||
REQUIRE(iters == 4.0);
|
||||
}
|
||||
}
|
||||
|
||||
#endif // _CCCL_CTK_AT_LEAST(12, 4)
|
||||
@@ -0,0 +1,237 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Regression test for the C-facade stackable-token dispatch fix. Combining a
|
||||
// stackable token with push_graph / pop_prologue used to abort inside STF with
|
||||
// a "Data interface type mismatch" (assumed void_interface, actual
|
||||
// mdspan<char, ..., layout_stride>) because the C API treated every stackable
|
||||
// logical-data handle as a slice<char> and mis-cast tokens. The abort was a
|
||||
// hard C-level abort, so the Python binding that drives this exact sequence
|
||||
// could not catch it:
|
||||
//
|
||||
// ctx = stf.stackable_context()
|
||||
// tok = ctx.token()
|
||||
// ctx.push()
|
||||
// with ctx.task(tok.write()): ...
|
||||
// with ctx.task(tok.read()): ...
|
||||
// step_graph = ctx.pop_prologue_shared()
|
||||
//
|
||||
// These tests drive the same sequences through the C stackable API directly,
|
||||
// so the path stays covered without any Python / Warp in the picture.
|
||||
//
|
||||
// The existing `stackable: token + fence` test uses tokens but outside any
|
||||
// push_graph scope, and the existing pop_prologue tests use real logical_data;
|
||||
// so the combination "tokens inside push_graph" is only exercised here.
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <cccl/c/experimental/stf/stf.h>
|
||||
|
||||
namespace
|
||||
{
|
||||
__global__ void noop_kernel() {}
|
||||
} // namespace
|
||||
|
||||
// Minimal case: a single token-only task inside a push_graph / pop scope.
|
||||
// Does NOT use pop_prologue — just push_graph + pop — so token task-deps
|
||||
// handling is covered independently of the prologue machinery.
|
||||
C2H_TEST("stackable: token in push_graph scope (no prologue)", "[stackable][token][bug]")
|
||||
{
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
stf_logical_data_handle tok = stf_stackable_token(ctx);
|
||||
REQUIRE(tok != nullptr);
|
||||
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, tok, STF_WRITE);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
noop_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>();
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
stf_stackable_pop(ctx);
|
||||
|
||||
stf_stackable_token_destroy(tok);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
// Exact mirror of the Python run_stf_unified path:
|
||||
// ctx.push() -> task(tok.write()) -> task(tok.read()) -> pop_prologue(_shared)
|
||||
// This used to abort inside pop_prologue(_shared)() with the void_interface vs
|
||||
// mdspan<char> mismatch before the C-facade dispatch fix.
|
||||
C2H_TEST("stackable: token write/read chain + pop_prologue", "[stackable][token][launchable][bug]")
|
||||
{
|
||||
const int relaunchN = 4;
|
||||
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
stf_logical_data_handle tok = stf_stackable_token(ctx);
|
||||
REQUIRE(tok != nullptr);
|
||||
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
// Writer task (equivalent of Python `tok.write()`).
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, tok, STF_WRITE);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
noop_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>();
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
|
||||
// Reader task (equivalent of Python `tok.read()`).
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, tok, STF_READ);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
noop_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>();
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
}
|
||||
|
||||
stf_launchable_graph_handle lh = stf_stackable_pop_prologue(ctx);
|
||||
REQUIRE(lh != nullptr);
|
||||
for (int k = 0; k < relaunchN; ++k)
|
||||
{
|
||||
stf_launchable_graph_launch(lh);
|
||||
}
|
||||
stf_stackable_pop_epilogue(ctx);
|
||||
stf_launchable_graph_destroy(lh);
|
||||
|
||||
stf_stackable_token_destroy(tok);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
// Same as above but using the shared flavour of pop_prologue, which is what
|
||||
// the Python `pop_prologue_shared()` binding calls into.
|
||||
C2H_TEST("stackable: token write/read chain + pop_prologue_shared", "[stackable][token][launchable][bug]")
|
||||
{
|
||||
const int relaunchN = 4;
|
||||
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
stf_logical_data_handle tok = stf_stackable_token(ctx);
|
||||
REQUIRE(tok != nullptr);
|
||||
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, tok, STF_WRITE);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
noop_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>();
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, tok, STF_READ);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
noop_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>();
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
}
|
||||
|
||||
stf_launchable_graph_shared h = nullptr;
|
||||
REQUIRE(stf_stackable_pop_prologue_shared(ctx, &h) == 0);
|
||||
REQUIRE(h != nullptr);
|
||||
for (int k = 0; k < relaunchN; ++k)
|
||||
{
|
||||
stf_launchable_graph_shared_launch(h);
|
||||
}
|
||||
// Last free drops the strong ref and runs pop_epilogue automatically.
|
||||
stf_launchable_graph_shared_free(h);
|
||||
|
||||
stf_stackable_token_destroy(tok);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
// Sanity check: replacing the token with a real logical_data in the same
|
||||
// push_graph + pop_prologue shape *should* work. This matches the
|
||||
// `run_stf_unified_ld` workaround that the Python mockup confirmed OK.
|
||||
C2H_TEST("stackable: logical_data write/read chain + pop_prologue (workaround)", "[stackable][launchable]")
|
||||
{
|
||||
const size_t N = 8;
|
||||
const int relaunchN = 4;
|
||||
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
uint8_t* host_dep = nullptr;
|
||||
cudaError_t err = cudaMallocHost(&host_dep, N * sizeof(uint8_t));
|
||||
REQUIRE(err == cudaSuccess);
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
host_dep[i] = 0;
|
||||
}
|
||||
|
||||
stf_logical_data_handle ld = stf_stackable_logical_data(ctx, host_dep, N * sizeof(uint8_t));
|
||||
REQUIRE(ld != nullptr);
|
||||
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, ld, STF_RW);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
noop_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>();
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, ld, STF_READ);
|
||||
stf_task_enable_capture(t);
|
||||
stf_task_start(t);
|
||||
noop_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>();
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
}
|
||||
|
||||
stf_launchable_graph_handle lh = stf_stackable_pop_prologue(ctx);
|
||||
REQUIRE(lh != nullptr);
|
||||
for (int k = 0; k < relaunchN; ++k)
|
||||
{
|
||||
stf_launchable_graph_launch(lh);
|
||||
}
|
||||
stf_stackable_pop_epilogue(ctx);
|
||||
stf_launchable_graph_destroy(lh);
|
||||
|
||||
stf_stackable_logical_data_destroy(ld);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
|
||||
REQUIRE(cudaFreeHost(host_dep) == cudaSuccess);
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
// Minimal tests for stf_ctx_create_ex() with has_stream=1 (caller-provided
|
||||
// CUDA stream) on the stream backend, with no async_resources handle shared
|
||||
// across contexts. These verify that contexts created back-to-back on the same
|
||||
// caller stream chain their work transitively through that stream.
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <cccl/c/experimental/stf/stf.h>
|
||||
|
||||
namespace
|
||||
{
|
||||
// A device sink that is written but never read. Publishing the busy-loop
|
||||
// result here gives the loop an observable side effect, so the compiler
|
||||
// cannot optimize it away, without perturbing the result buffer.
|
||||
__device__ unsigned g_busy_sink;
|
||||
|
||||
// Writes `value` into every slot of `arr`. The inner busy loop widens the
|
||||
// kernel window so that a failure to chain ctx2-after-ctx1 is observable:
|
||||
// ctx1 is still running when ctx2's kernel races in.
|
||||
__global__ void slow_set_kernel(int* arr, int n, int value, int iters)
|
||||
{
|
||||
const int tid = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
|
||||
if (tid >= n)
|
||||
{
|
||||
return;
|
||||
}
|
||||
// Busy loop to keep the kernel resident on the SM for a while. `acc` is
|
||||
// unsigned so the accumulation wraps with well-defined behavior.
|
||||
unsigned acc = 0;
|
||||
for (int i = 0; i < iters; ++i)
|
||||
{
|
||||
acc += (static_cast<unsigned>(i) * 1103515245u + 12345u) & 0x7fffffffu;
|
||||
}
|
||||
// Publish `acc` via an atomic: an observable, race-free side effect that
|
||||
// keeps the loop alive while the stored result stays exactly `value`.
|
||||
atomicAdd(&g_busy_sink, acc);
|
||||
arr[tid] = value;
|
||||
}
|
||||
|
||||
void submit_set(stf_ctx_handle ctx, int* d_arr, int n, int value, int iters)
|
||||
{
|
||||
stf_logical_data_handle tok = stf_token(ctx);
|
||||
REQUIRE(tok != nullptr);
|
||||
stf_logical_data_set_symbol(tok, "tok");
|
||||
|
||||
stf_task_handle t = stf_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_task_set_symbol(t, "slow_set");
|
||||
stf_task_add_dep(t, tok, STF_RW);
|
||||
stf_task_start(t);
|
||||
|
||||
CUstream s = stf_task_get_custream(t);
|
||||
REQUIRE(s != nullptr);
|
||||
|
||||
const int threads = 128;
|
||||
const int blocks = (n + threads - 1) / threads;
|
||||
slow_set_kernel<<<blocks, threads, 0, (cudaStream_t) s>>>(d_arr, n, value, iters);
|
||||
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
|
||||
stf_logical_data_destroy(tok);
|
||||
}
|
||||
|
||||
// Submits `K` concurrent token-tasks in a single context; each writes `value`
|
||||
// into its own slice of `d_arr`. Multiple independent tokens per context make
|
||||
// STF spread kernels across several pool streams, so ordering depends on the
|
||||
// caller-stream chaining contract.
|
||||
void run_ctx_k_concurrent(cudaStream_t s, int* d_arr, int N, int K, int value, int iters)
|
||||
{
|
||||
stf_ctx_options opts{};
|
||||
opts.backend = STF_BACKEND_STREAM;
|
||||
opts.has_stream = 1;
|
||||
opts.stream = s;
|
||||
opts.handle = nullptr;
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create_ex(&opts);
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
const int per = N / K;
|
||||
for (int k = 0; k < K; ++k)
|
||||
{
|
||||
stf_logical_data_handle tok = stf_token(ctx);
|
||||
REQUIRE(tok != nullptr);
|
||||
|
||||
stf_task_handle t = stf_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_task_add_dep(t, tok, STF_RW);
|
||||
stf_task_start(t);
|
||||
|
||||
CUstream ts = stf_task_get_custream(t);
|
||||
const int threads = 128;
|
||||
const int blocks = (per + threads - 1) / threads;
|
||||
int* slice = d_arr + k * per;
|
||||
slow_set_kernel<<<blocks, threads, 0, (cudaStream_t) ts>>>(slice, per, value, iters);
|
||||
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
stf_logical_data_destroy(tok);
|
||||
}
|
||||
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
// More faithful MLP mimic: K concurrent tokens, each with T chained tasks
|
||||
// (sequential RW on the same token), so each token effectively owns a chain of
|
||||
// T slow kernels on one pool stream.
|
||||
void run_ctx_k_chains(cudaStream_t s, int* d_arr, int N, int K, int chain_len, int value, int iters)
|
||||
{
|
||||
stf_ctx_options opts{};
|
||||
opts.backend = STF_BACKEND_STREAM;
|
||||
opts.has_stream = 1;
|
||||
opts.stream = s;
|
||||
opts.handle = nullptr;
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create_ex(&opts);
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
const int per = N / K;
|
||||
std::vector<stf_logical_data_handle> toks(K);
|
||||
for (int k = 0; k < K; ++k)
|
||||
{
|
||||
toks[k] = stf_token(ctx);
|
||||
REQUIRE(toks[k] != nullptr);
|
||||
}
|
||||
|
||||
for (int step = 0; step < chain_len; ++step)
|
||||
{
|
||||
for (int k = 0; k < K; ++k)
|
||||
{
|
||||
stf_task_handle t = stf_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_task_add_dep(t, toks[k], STF_RW);
|
||||
stf_task_start(t);
|
||||
|
||||
CUstream ts = stf_task_get_custream(t);
|
||||
const int threads = 128;
|
||||
const int blocks = (per + threads - 1) / threads;
|
||||
int* slice = d_arr + k * per;
|
||||
slow_set_kernel<<<blocks, threads, 0, (cudaStream_t) ts>>>(slice, per, value, iters);
|
||||
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
}
|
||||
|
||||
for (int k = 0; k < K; ++k)
|
||||
{
|
||||
stf_logical_data_destroy(toks[k]);
|
||||
}
|
||||
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("stf_ctx_create_ex: 1 token per context, back-to-back, stream-only", "[context][stream]")
|
||||
{
|
||||
constexpr int N = 1 << 14;
|
||||
constexpr int ITERS = 1 << 18;
|
||||
|
||||
cudaStream_t s{};
|
||||
REQUIRE(cudaStreamCreate(&s) == cudaSuccess);
|
||||
|
||||
int* d_arr = nullptr;
|
||||
REQUIRE(cudaMalloc(&d_arr, N * sizeof(int)) == cudaSuccess);
|
||||
REQUIRE(cudaMemsetAsync(d_arr, 0, N * sizeof(int), s) == cudaSuccess);
|
||||
|
||||
for (int iter = 0; iter < 20; ++iter)
|
||||
{
|
||||
{
|
||||
stf_ctx_options opts{};
|
||||
opts.backend = STF_BACKEND_STREAM;
|
||||
opts.has_stream = 1;
|
||||
opts.stream = s;
|
||||
opts.handle = nullptr;
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create_ex(&opts);
|
||||
REQUIRE(ctx != nullptr);
|
||||
submit_set(ctx, d_arr, N, /*value=*/1, ITERS);
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
{
|
||||
stf_ctx_options opts{};
|
||||
opts.backend = STF_BACKEND_STREAM;
|
||||
opts.has_stream = 1;
|
||||
opts.stream = s;
|
||||
opts.handle = nullptr;
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create_ex(&opts);
|
||||
REQUIRE(ctx != nullptr);
|
||||
submit_set(ctx, d_arr, N, /*value=*/2, ITERS);
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
|
||||
REQUIRE(cudaStreamSynchronize(s) == cudaSuccess);
|
||||
int h_arr[16]{};
|
||||
REQUIRE(cudaMemcpy(h_arr, d_arr, sizeof(h_arr), cudaMemcpyDeviceToHost) == cudaSuccess);
|
||||
for (int i = 0; i < static_cast<int>(sizeof(h_arr) / sizeof(int)); ++i)
|
||||
{
|
||||
INFO("iter=" << iter << " i=" << i << " value=" << h_arr[i]);
|
||||
REQUIRE(h_arr[i] == 2);
|
||||
}
|
||||
}
|
||||
|
||||
REQUIRE(cudaFree(d_arr) == cudaSuccess);
|
||||
REQUIRE(cudaStreamDestroy(s) == cudaSuccess);
|
||||
}
|
||||
|
||||
C2H_TEST("stf_ctx_create_ex: K chains of T tasks per token, back-to-back, stream-only, no handle",
|
||||
"[context][stream][tokens][lifetime]")
|
||||
{
|
||||
constexpr int N = 1 << 16;
|
||||
constexpr int K = 8;
|
||||
constexpr int CHAIN_LEN = 20;
|
||||
constexpr int ITERS = 1 << 18;
|
||||
|
||||
cudaStream_t s{};
|
||||
REQUIRE(cudaStreamCreate(&s) == cudaSuccess);
|
||||
|
||||
int* d_arr = nullptr;
|
||||
REQUIRE(cudaMalloc(&d_arr, N * sizeof(int)) == cudaSuccess);
|
||||
REQUIRE(cudaMemsetAsync(d_arr, 0, N * sizeof(int), s) == cudaSuccess);
|
||||
|
||||
for (int iter = 0; iter < 20; ++iter)
|
||||
{
|
||||
run_ctx_k_chains(s, d_arr, N, K, CHAIN_LEN, /*value=*/1, ITERS);
|
||||
run_ctx_k_chains(s, d_arr, N, K, CHAIN_LEN, /*value=*/2, ITERS);
|
||||
|
||||
REQUIRE(cudaStreamSynchronize(s) == cudaSuccess);
|
||||
std::vector<int> h_arr(N, 0);
|
||||
REQUIRE(cudaMemcpy(h_arr.data(), d_arr, N * sizeof(int), cudaMemcpyDeviceToHost) == cudaSuccess);
|
||||
|
||||
int mismatches = 0;
|
||||
int first_bad_i = -1;
|
||||
int first_bad_v = 0;
|
||||
for (int i = 0; i < N; ++i)
|
||||
{
|
||||
if (h_arr[i] != 2)
|
||||
{
|
||||
++mismatches;
|
||||
if (first_bad_i < 0)
|
||||
{
|
||||
first_bad_i = i;
|
||||
first_bad_v = h_arr[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
INFO("iter=" << iter << " mismatches=" << mismatches << " first_bad_idx=" << first_bad_i
|
||||
<< " first_bad_val=" << first_bad_v);
|
||||
REQUIRE(mismatches == 0);
|
||||
}
|
||||
|
||||
REQUIRE(cudaFree(d_arr) == cudaSuccess);
|
||||
REQUIRE(cudaStreamDestroy(s) == cudaSuccess);
|
||||
}
|
||||
|
||||
C2H_TEST("stf_ctx_create_ex: K concurrent tokens per context, back-to-back, stream-only", "[context][stream][tokens]")
|
||||
{
|
||||
constexpr int N = 1 << 16;
|
||||
constexpr int K = 8;
|
||||
constexpr int ITERS = 1 << 18;
|
||||
|
||||
cudaStream_t s{};
|
||||
REQUIRE(cudaStreamCreate(&s) == cudaSuccess);
|
||||
|
||||
int* d_arr = nullptr;
|
||||
REQUIRE(cudaMalloc(&d_arr, N * sizeof(int)) == cudaSuccess);
|
||||
REQUIRE(cudaMemsetAsync(d_arr, 0, N * sizeof(int), s) == cudaSuccess);
|
||||
|
||||
for (int iter = 0; iter < 20; ++iter)
|
||||
{
|
||||
run_ctx_k_concurrent(s, d_arr, N, K, /*value=*/1, ITERS);
|
||||
run_ctx_k_concurrent(s, d_arr, N, K, /*value=*/2, ITERS);
|
||||
|
||||
REQUIRE(cudaStreamSynchronize(s) == cudaSuccess);
|
||||
std::vector<int> h_arr(N, 0);
|
||||
REQUIRE(cudaMemcpy(h_arr.data(), d_arr, N * sizeof(int), cudaMemcpyDeviceToHost) == cudaSuccess);
|
||||
|
||||
int mismatches = 0;
|
||||
int first_bad_i = -1;
|
||||
int first_bad_v = 0;
|
||||
for (int i = 0; i < N; ++i)
|
||||
{
|
||||
if (h_arr[i] != 2)
|
||||
{
|
||||
++mismatches;
|
||||
if (first_bad_i < 0)
|
||||
{
|
||||
first_bad_i = i;
|
||||
first_bad_v = h_arr[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
INFO("iter=" << iter << " mismatches=" << mismatches << " first_bad_idx=" << first_bad_i
|
||||
<< " first_bad_val=" << first_bad_v);
|
||||
REQUIRE(mismatches == 0);
|
||||
}
|
||||
|
||||
REQUIRE(cudaFree(d_arr) == cudaSuccess);
|
||||
REQUIRE(cudaStreamDestroy(s) == cudaSuccess);
|
||||
}
|
||||
80
cccl_upstream/c/experimental/stf/test/test_task.cpp
Normal file
80
cccl_upstream/c/experimental/stf/test/test_task.cpp
Normal file
@@ -0,0 +1,80 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <cccl/c/experimental/stf/stf.h>
|
||||
|
||||
C2H_TEST("empty stf tasks", "[task]")
|
||||
{
|
||||
size_t N = 1000000;
|
||||
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
std::vector<float> X(N);
|
||||
std::vector<float> Y(N);
|
||||
std::vector<float> Z(N);
|
||||
|
||||
stf_logical_data_handle lX = stf_logical_data(ctx, X.data(), N * sizeof(float));
|
||||
stf_logical_data_handle lY = stf_logical_data(ctx, Y.data(), N * sizeof(float));
|
||||
stf_logical_data_handle lZ = stf_logical_data(ctx, Z.data(), N * sizeof(float));
|
||||
REQUIRE(lX != nullptr);
|
||||
REQUIRE(lY != nullptr);
|
||||
REQUIRE(lZ != nullptr);
|
||||
|
||||
stf_logical_data_set_symbol(lX, "X");
|
||||
stf_logical_data_set_symbol(lY, "Y");
|
||||
stf_logical_data_set_symbol(lZ, "Z");
|
||||
|
||||
stf_task_handle t1 = stf_task_create(ctx);
|
||||
REQUIRE(t1 != nullptr);
|
||||
stf_task_set_symbol(t1, "T1");
|
||||
stf_task_add_dep(t1, lX, STF_RW);
|
||||
stf_task_start(t1);
|
||||
stf_task_end(t1);
|
||||
stf_task_destroy(t1);
|
||||
|
||||
stf_task_handle t2 = stf_task_create(ctx);
|
||||
REQUIRE(t2 != nullptr);
|
||||
stf_task_set_symbol(t2, "T2");
|
||||
stf_task_add_dep(t2, lX, STF_READ);
|
||||
stf_task_add_dep(t2, lY, STF_RW);
|
||||
stf_task_start(t2);
|
||||
stf_task_end(t2);
|
||||
stf_task_destroy(t2);
|
||||
|
||||
stf_task_handle t3 = stf_task_create(ctx);
|
||||
REQUIRE(t3 != nullptr);
|
||||
stf_task_set_symbol(t3, "T3");
|
||||
stf_task_add_dep(t3, lX, STF_READ);
|
||||
stf_task_add_dep(t3, lZ, STF_RW);
|
||||
stf_task_start(t3);
|
||||
stf_task_end(t3);
|
||||
stf_task_destroy(t3);
|
||||
|
||||
stf_task_handle t4 = stf_task_create(ctx);
|
||||
REQUIRE(t4 != nullptr);
|
||||
stf_task_set_symbol(t4, "T4");
|
||||
stf_task_add_dep(t4, lY, STF_READ);
|
||||
stf_task_add_dep(t4, lZ, STF_RW);
|
||||
stf_task_start(t4);
|
||||
stf_task_end(t4);
|
||||
stf_task_destroy(t4);
|
||||
|
||||
stf_logical_data_destroy(lX);
|
||||
stf_logical_data_destroy(lY);
|
||||
stf_logical_data_destroy(lZ);
|
||||
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
94
cccl_upstream/c/experimental/stf/test/test_task_get_graph.cu
Normal file
94
cccl_upstream/c/experimental/stf/test/test_task_get_graph.cu
Normal file
@@ -0,0 +1,94 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <cccl/c/experimental/stf/stf.h>
|
||||
|
||||
__global__ void scale_kernel(int cnt, double* data, double factor)
|
||||
{
|
||||
const int tid = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
|
||||
const int nthreads = static_cast<int>(gridDim.x * blockDim.x);
|
||||
for (int i = tid; i < cnt; i += nthreads)
|
||||
{
|
||||
data[i] *= factor;
|
||||
}
|
||||
}
|
||||
|
||||
// Exercise the explicit-graph path: instead of capturing a stream with
|
||||
// stf_task_enable_capture() + stf_task_get_custream(), an expert caller fetches
|
||||
// the task's child cudaGraph_t with stf_task_get_graph() and adds nodes into it
|
||||
// directly (here a single kernel node). STF wires the task's dependencies around
|
||||
// the child graph.
|
||||
C2H_TEST("task_get_graph: explicit kernel node in a stackable graph scope", "[stackable][task_get_graph]")
|
||||
{
|
||||
const size_t N = 256;
|
||||
|
||||
stf_ctx_handle ctx = stf_stackable_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
double* host_data;
|
||||
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
host_data[i] = static_cast<double>(i);
|
||||
}
|
||||
|
||||
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
|
||||
REQUIRE(lA != nullptr);
|
||||
|
||||
// Multiply by 3 inside a nested graph scope using an explicitly added kernel node.
|
||||
stf_stackable_push_graph(ctx);
|
||||
{
|
||||
stf_task_handle t = stf_stackable_task_create(ctx);
|
||||
REQUIRE(t != nullptr);
|
||||
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
|
||||
// Note: no stf_task_enable_capture() here -- the explicit-graph path is
|
||||
// mutually exclusive with stream capture.
|
||||
stf_task_start(t);
|
||||
|
||||
cudaGraph_t g = stf_task_get_graph(t);
|
||||
REQUIRE(g != nullptr);
|
||||
|
||||
double* d = static_cast<double*>(stf_task_get(t, 0));
|
||||
int n = static_cast<int>(N);
|
||||
double f = 3.0;
|
||||
void* kernel_args[] = {&n, &d, &f};
|
||||
|
||||
cudaKernelNodeParams kparams = {};
|
||||
kparams.func = reinterpret_cast<void*>(&scale_kernel);
|
||||
kparams.gridDim = dim3(2, 1, 1);
|
||||
kparams.blockDim = dim3(64, 1, 1);
|
||||
kparams.sharedMemBytes = 0;
|
||||
kparams.kernelParams = kernel_args;
|
||||
kparams.extra = nullptr;
|
||||
|
||||
cudaGraphNode_t node;
|
||||
REQUIRE(cudaGraphAddKernelNode(&node, g, nullptr, 0, &kparams) == cudaSuccess);
|
||||
|
||||
stf_task_end(t);
|
||||
stf_task_destroy(t);
|
||||
}
|
||||
stf_stackable_pop(ctx);
|
||||
|
||||
stf_stackable_logical_data_destroy(lA);
|
||||
stf_stackable_ctx_finalize(ctx);
|
||||
|
||||
for (size_t i = 0; i < N; i++)
|
||||
{
|
||||
REQUIRE(std::fabs(host_data[i] - 3.0 * static_cast<double>(i)) < 1e-10);
|
||||
}
|
||||
|
||||
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
|
||||
}
|
||||
72
cccl_upstream/c/experimental/stf/test/test_token.cpp
Normal file
72
cccl_upstream/c/experimental/stf/test/test_token.cpp
Normal file
@@ -0,0 +1,72 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <cccl/c/experimental/stf/stf.h>
|
||||
|
||||
C2H_TEST("stf token", "[token]")
|
||||
{
|
||||
stf_ctx_handle ctx = stf_ctx_create();
|
||||
REQUIRE(ctx != nullptr);
|
||||
|
||||
stf_logical_data_handle lX = stf_token(ctx);
|
||||
stf_logical_data_handle lY = stf_token(ctx);
|
||||
stf_logical_data_handle lZ = stf_token(ctx);
|
||||
REQUIRE(lX != nullptr);
|
||||
REQUIRE(lY != nullptr);
|
||||
REQUIRE(lZ != nullptr);
|
||||
|
||||
stf_logical_data_set_symbol(lX, "X");
|
||||
stf_logical_data_set_symbol(lY, "Y");
|
||||
stf_logical_data_set_symbol(lZ, "Z");
|
||||
|
||||
stf_task_handle t1 = stf_task_create(ctx);
|
||||
REQUIRE(t1 != nullptr);
|
||||
stf_task_set_symbol(t1, "T1");
|
||||
stf_task_add_dep(t1, lX, STF_RW);
|
||||
stf_task_start(t1);
|
||||
stf_task_end(t1);
|
||||
stf_task_destroy(t1);
|
||||
|
||||
stf_task_handle t2 = stf_task_create(ctx);
|
||||
REQUIRE(t2 != nullptr);
|
||||
stf_task_set_symbol(t2, "T2");
|
||||
stf_task_add_dep(t2, lX, STF_READ);
|
||||
stf_task_add_dep(t2, lY, STF_RW);
|
||||
stf_task_start(t2);
|
||||
stf_task_end(t2);
|
||||
stf_task_destroy(t2);
|
||||
|
||||
stf_task_handle t3 = stf_task_create(ctx);
|
||||
REQUIRE(t3 != nullptr);
|
||||
stf_task_set_symbol(t3, "T3");
|
||||
stf_task_add_dep(t3, lX, STF_READ);
|
||||
stf_task_add_dep(t3, lZ, STF_RW);
|
||||
stf_task_start(t3);
|
||||
stf_task_end(t3);
|
||||
stf_task_destroy(t3);
|
||||
|
||||
stf_task_handle t4 = stf_task_create(ctx);
|
||||
REQUIRE(t4 != nullptr);
|
||||
stf_task_set_symbol(t4, "T4");
|
||||
stf_task_add_dep(t4, lY, STF_READ);
|
||||
stf_task_add_dep(t4, lZ, STF_RW);
|
||||
stf_task_start(t4);
|
||||
stf_task_end(t4);
|
||||
stf_task_destroy(t4);
|
||||
|
||||
stf_logical_data_destroy(lX);
|
||||
stf_logical_data_destroy(lY);
|
||||
stf_logical_data_destroy(lZ);
|
||||
|
||||
stf_ctx_finalize(ctx);
|
||||
}
|
||||
Reference in New Issue
Block a user