dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

Browse Source 
CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0
This commit is contained in:
EngineX CI
2026-07-30 09:35:51 +00:00
parent b4d01f481e
commit 56fd68e7dd
8871 changed files with 1454674 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
cccl_upstream/CMakeLists.txt Normal file
View File

@@ -0,0 +1,163 @@
# 3.18 is the minimum for including the project with add_subdirectory.
# 3.21 is the minimum for the developer build.
cmake_minimum_required(VERSION 3.18)
# sccache cannot handle the -Fd option generating pdb files:
if (POLICY CMP0141)
cmake_policy(SET CMP0141 NEW)
endif()
# Determine whether CCCL is the top-level project or included into
# another project via add_subdirectory()
if (NOT DEFINED CCCL_TOPLEVEL_PROJECT)
# DO NOT REMOVE THE FOLLOWING LINE
set(CCCL_TOPLEVEL_PROJECT OFF)
# REQUIRED FOR CCCL TO WORK VIA CPM WITH INSTALL RULES
if ("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_LIST_DIR}")
set(CCCL_TOPLEVEL_PROJECT ON)
endif()
endif()
# Enable CXX so CMake can configure install paths
project(CCCL LANGUAGES CXX)
# Enable CCCL specific platform files
list(PREPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
include(cmake/CCCLInstallRules.cmake)
# Support adding CCCL to a parent project via add_subdirectory.
include(cmake/CCCLAddSubdirHelper.cmake) # Always include, this is used by subprojects as well.
if (NOT CCCL_TOPLEVEL_PROJECT)
include(cmake/CCCLAddSubdir.cmake)
endif()
if (CCCL_TOPLEVEL_PROJECT AND NOT CCCL_SKIP_BUILD_CHECKS)
# We require a higher cmake version for dev builds
cmake_minimum_required(VERSION 3.21)
# Handle special CCCL values for CMAKE_CUDA_ARCHITECTURES
include(cmake/CCCLCheckCudaArchitectures.cmake)
cccl_check_cuda_architectures()
# Perform developer build checks
include(cmake/CCCLDevBuildChecks.cmake)
cccl_dev_build_checks()
endif()
option(CCCL_ENABLE_LIBCUDACXX "Enable the libcu++ developer build." OFF)
option(CCCL_ENABLE_CUB "Enable the CUB developer build." OFF)
option(CCCL_ENABLE_THRUST "Enable the Thrust developer build." OFF)
option(CCCL_ENABLE_TESTING "Enable CUDA C++ Core Library tests." OFF)
option(CCCL_ENABLE_EXAMPLES "Enable CUDA C++ Core Library examples." OFF)
option(CCCL_ENABLE_C_PARALLEL "Enable CUDA C Parallel Library." OFF)
option(
CCCL_ENABLE_C_PARALLEL_V2
"Enable CUDA C Parallel Library v2 (HostJIT-based)."
OFF
)
option(CCCL_ENABLE_C_EXPERIMENTAL_STF "Enable CUDA C CUDASTF Library." OFF)
option(CCCL_ENABLE_NVBENCH_HELPER "Enable the NVBench Helper Dev Build." OFF)
if ("NVHPC" STREQUAL "${CMAKE_CXX_COMPILER_ID}")
set(CCCL_ENABLE_BENCHMARKS OFF)
else()
option(CCCL_ENABLE_BENCHMARKS "Enable CUDA C++ Core Library benchmarks." OFF)
endif()
option(CCCL_ENABLE_TILE "Enable tile support" OFF)
option(
CCCL_ENABLE_UNSTABLE
"Enable targets and developer build options for unstable projects."
OFF
)
if (CCCL_ENABLE_UNSTABLE)
option(
CCCL_ENABLE_CUDAX
"Enable the CUDA Experimental developer build."
${CCCL_TOPLEVEL_PROJECT}
)
else()
# Always off if unstable disabled:
# Note that this doesn't override the cache variable, but rather creates a new
# directory-scoped variable that shadows it. This is sufficient for our purposes.
set(CCCL_ENABLE_CUDAX OFF)
endif()
option(CCCL_ENABLE_CLANG_TIDY "Enable clang-tidy" OFF)
mark_as_advanced(CCCL_ENABLE_CLANG_TIDY)
if (CCCL_TOPLEVEL_PROJECT)
include(cmake/CCCLAddTidyTarget.cmake)
if (CCCL_ENABLE_CLANG_TIDY)
cccl_tidy_init()
endif()
endif()
include(CTest)
enable_testing()
option(CCCL_USE_LIBCXX "Use libc++ when compiling and linking" OFF)
if (CCCL_TOPLEVEL_PROJECT)
include(cmake/AppendOptionIfAvailable.cmake)
include(cmake/CCCLUtilities.cmake) # include this before other CCCL helpers
include(cmake/CCCLAddExecutable.cmake)
include(cmake/CCCLBuildCompilerTargets.cmake)
include(cmake/CCCLClangdCompileInfo.cmake)
include(cmake/CCCLConfigureTarget.cmake)
include(cmake/CCCLEnsureMetaTargets.cmake)
include(cmake/CCCLGenerateHeaderTests.cmake)
include(cmake/CCCLGetDependencies.cmake)
include(cmake/CCCLTestParams.cmake)
cccl_build_compiler_targets()
endif()
option(
CCCL_ENABLE_CUDA_SMOKE_TESTS
"Build the CUDA runtime smoke test (cccl.test.cuda_runtime_smoke)."
OFF
)
set(CCCL_CUDA_SMOKE_TARGET cccl.test.cuda_runtime_smoke)
add_subdirectory(libcudacxx)
add_subdirectory(cub)
add_subdirectory(thrust)
if (CCCL_ENABLE_UNSTABLE)
add_subdirectory(cudax)
endif()
if (
CCCL_ENABLE_C_PARALLEL
OR CCCL_ENABLE_C_PARALLEL_V2
OR CCCL_ENABLE_C_EXPERIMENTAL_STF
)
add_subdirectory(c)
endif()
if (CCCL_ENABLE_TESTING)
add_subdirectory("ci")
add_subdirectory("test")
endif()
if (CCCL_ENABLE_CUDA_SMOKE_TESTS AND CCCL_TOPLEVEL_PROJECT)
add_subdirectory(test/cuda_smoke)
endif()
if (CCCL_ENABLE_EXAMPLES)
add_subdirectory(examples)
endif()
if (CCCL_ENABLE_NVBENCH_HELPER)
add_subdirectory(nvbench_helper)
endif()
# Must stay at the end of this file.
include(cmake/CCCLHideThirdPartyOptions.cmake)

410
cccl_upstream/LICENSE Normal file
View File

@@ -0,0 +1,410 @@
==============================================================================
Thrust is under the Apache Licence v2.0, with some specific exceptions listed below
libcu++ is under the Apache License v2.0 with LLVM Exceptions:
==============================================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================
Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy):
==============================================================================
---- LLVM Exceptions to the Apache 2.0 License ----
As an exception, if, as a result of your compiling your source code, portions
of this Software are embedded into an Object form of such source code, you
may redistribute such embedded portions in such Object form without complying
with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
In addition, if you combine or link compiled forms of this Software with
software that is licensed under the GPLv2 ("Combined Software") and if a
court of competent jurisdiction determines that the patent provision (Section
3), the indemnity provision (Section 9) or other Section of the License
conflicts with the conditions of the GPLv2, you may retroactively and
prospectively choose to deem waived or otherwise exclude such Section(s) of
the License, but only in their entirety and only with respect to the Combined
Software.
==============================================================================
Software from third parties included in the LLVM Project:
==============================================================================
The LLVM Project contains third party software which is under different license
terms. All such code will be identified clearly using at least one of two
mechanisms:
1) It will be in a separate directory tree with its own `LICENSE.txt` or
`LICENSE` file at the top containing the specific license and restrictions
which apply to that software, or
2) It will contain specific license and restriction terms at the top of every
file.
==============================================================================
Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy):
==============================================================================
The libc++ library is dual licensed under both the University of Illinois
"BSD-Like" license and the MIT license. As a user of this code you may choose
to use it under either license. As a contributor, you agree to allow your code
to be used under both.
Full text of the relevant licenses is included below.
==============================================================================
University of Illinois/NCSA
Open Source License
Copyright (c) 2009-2019 by the contributors listed in CREDITS.TXT
All rights reserved.
Developed by:
LLVM Team
University of Illinois at Urbana-Champaign
http://llvm.org
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal with
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimers.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimers in the
documentation and/or other materials provided with the distribution.
* Neither the names of the LLVM Team, University of Illinois at
Urbana-Champaign, nor the names of its contributors may be used to
endorse or promote products derived from this Software without specific
prior written permission.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
SOFTWARE.
==============================================================================
Copyright (c) 2009-2014 by the contributors listed in CREDITS.TXT
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
================================================================================
Some portions of Thrust may be licensed under other compatible open-source
licenses. Any divergence from the Apache 2 license will be noted in the source
code where applicable.
Portions under other terms include, but are not limited to:
================================================================================
Various C++ utility classes in Thrust are based on the Boost Iterator, Tuple,
System, and Random Number libraries, which are provided under the Boost Software
License:
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
================================================================================
Portions of the thrust::complex implementation are derived from FreeBSD with the
following terms:
================================================================================
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice[1] unmodified, this list of conditions, and the following
disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
[1] Individual copyright notices from the original authors are included in
the relevant source files.
==============================================================================
CUB's source code is released under the BSD 3-Clause license:
==============================================================================
Copyright (c) 2010-2011, Duane Merrill. All rights reserved.
Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the NVIDIA CORPORATION nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

522
cccl_upstream/README.md Normal file
View File

@@ -0,0 +1,522 @@
[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/NVIDIA/cccl?quickstart=1&devcontainer_path=.devcontainer%2Fdevcontainer.json)
|[Contributor Guide](https://github.com/NVIDIA/cccl/blob/main/CONTRIBUTING.md)|[Dev Containers](https://github.com/NVIDIA/cccl/blob/main/.devcontainer/README.md)|[Discord](https://discord.gg/nvidiadeveloper)|[Godbolt](https://godbolt.org/z/x4G73af9a)|[GitHub Project](https://github.com/orgs/NVIDIA/projects/6)|[Documentation](https://nvidia.github.io/cccl)|
|-|-|-|-|-|-|
# CUDA Core Compute Libraries (CCCL)
Welcome to the CUDA Core Compute Libraries (CCCL) where our mission is to make CUDA more delightful.
This repository unifies three essential CUDA C++ libraries into a single, convenient repository:
- [Thrust](thrust) ([former repo](https://github.com/nvidia/thrust))
- [CUB](cub) ([former repo](https://github.com/nvidia/cub))
- [libcudacxx](libcudacxx) ([former repo](https://github.com/nvidia/libcudacxx))
The goal of CCCL is to provide CUDA C++ developers with building blocks that make it easier to write safe and efficient code.
Bringing these libraries together streamlines your development process and broadens your ability to leverage the power of CUDA C++.
For more information about the decision to unify these projects, see the [announcement here](https://github.com/NVIDIA/cccl/discussions/520).
## Overview
The concept for the CUDA Core Compute Libraries (CCCL) grew organically out of the Thrust, CUB, and libcudacxx projects that were developed independently over the years with a similar goal: to provide high-quality, high-performance, and easy-to-use C++ abstractions for CUDA developers.
Naturally, there was a lot of overlap among the three projects, and it became clear the community would be better served by unifying them into a single repository.
- **Thrust** is the C++ parallel algorithms library which inspired the introduction of parallel algorithms to the C++ Standard Library. Thrust's high-level interface greatly enhances programmer productivity while enabling performance portability between GPUs and multicore CPUs via configurable backends that allow using multiple parallel programming frameworks (such as CUDA, TBB, and OpenMP).
- **CUB** is a lower-level, CUDA-specific library designed for speed-of-light parallel algorithms across all GPU architectures. In addition to device-wide algorithms, it provides *cooperative algorithms* like block-wide reduction and warp-wide scan, providing CUDA kernel developers with building blocks to create speed-of-light, custom kernels.
- **libcudacxx** is the CUDA C++ Standard Library. It provides an implementation of the C++ Standard Library that works in both host and device code. Additionally, it provides abstractions for CUDA-specific hardware features like synchronization primitives, cache control, atomics, and more.
The main goal of CCCL is to fill a similar role that the Standard C++ Library fills for Standard C++: provide general-purpose, speed-of-light tools to CUDA C++ developers, allowing them to focus on solving the problems that matter.
Unifying these projects is the first step towards realizing that goal.
## Example
This is a simple example demonstrating the use of CCCL functionality from Thrust, CUB, and libcudacxx.
It shows how to use Thrust/CUB/libcudacxx to implement a simple parallel reduction kernel.
Each thread block computes the sum of a subset of the array using `cub::BlockReduce`.
The sum of each block is then reduced to a single value using an atomic add via `cuda::atomic_ref` from libcudacxx.
It then shows how the same reduction can be done using Thrust's `reduce` algorithm and compares the results.
[Try it live on Godbolt!](https://godbolt.org/z/3KaWz3Msf)
```cpp
#include <thrust/execution_policy.h>
#include <thrust/device_vector.h>
#include <cub/block/block_reduce.cuh>
#include <cuda/atomic>
#include <cuda/cmath>
#include <cuda/std/span>
#include <cstdio>
template <int block_size>
__global__ void reduce(cuda::std::span<int const> data, cuda::std::span<int> result) {
using BlockReduce = cub::BlockReduce<int, block_size>;
__shared__ typename BlockReduce::TempStorage temp_storage;
int const index = threadIdx.x + blockIdx.x * blockDim.x;
int sum = 0;
if (index < data.size()) {
sum += data[index];
}
sum = BlockReduce(temp_storage).Sum(sum);
if (threadIdx.x == 0) {
cuda::atomic_ref<int, cuda::thread_scope_device> atomic_result(result.front());
atomic_result.fetch_add(sum, cuda::memory_order_relaxed);
}
}
int main() {
// Allocate and initialize input data
int const N = 1000;
thrust::device_vector<int> data(N);
thrust::fill(data.begin(), data.end(), 1);
// Allocate output data
thrust::device_vector<int> kernel_result(1);
// Compute the sum reduction of `data` using a custom kernel
constexpr int block_size = 256;
int const num_blocks = cuda::ceil_div(N, block_size);
reduce<block_size><<<num_blocks, block_size>>>(cuda::std::span<int const>(thrust::raw_pointer_cast(data.data()), data.size()),
cuda::std::span<int>(thrust::raw_pointer_cast(kernel_result.data()), 1));
auto const err = cudaDeviceSynchronize();
if (err != cudaSuccess) {
std::cout << "Error: " << cudaGetErrorString(err) << '\n';
return -1;
}
int const custom_result = kernel_result[0];
// Compute the same sum reduction using Thrust
int const thrust_result = thrust::reduce(thrust::device, data.begin(), data.end(), 0);
// Ensure the two solutions are identical
std::printf("Custom kernel sum: %d\n", custom_result);
std::printf("Thrust reduce sum: %d\n", thrust_result);
assert(kernel_result[0] == thrust_result);
return 0;
}
```
## Getting Started
### Users
Everything in CCCL is header-only.
Therefore, users need only concern themselves with how they get the header files and how they incorporate them into their build system.
#### CUDA Toolkit
The easiest way to get started using CCCL is via the [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit) which includes the CCCL headers.
When you compile with `nvcc`, it automatically adds CCCL headers to your include path so you can simply `#include` any CCCL header in your code with no additional configuration required.
If compiling with another compiler, you will need to update your build system's include search path to point to the CCCL headers in your CTK install (e.g., `/usr/local/cuda/include`).
```cpp
#include <thrust/device_vector.h>
#include <cub/cub.cuh>
#include <cuda/std/atomic>
```
#### GitHub
Users who want to stay on the cutting edge of CCCL development are encouraged to use CCCL from GitHub.
Using a newer version of CCCL with an older version of the CUDA Toolkit is supported, but not the other way around.
For complete information on compatibility between CCCL and the CUDA Toolkit, see [our platform support](#platform-support).
Everything in CCCL is header-only, so cloning and including it in a simple project is as easy as the following:
```bash
git clone https://github.com/NVIDIA/cccl.git
nvcc -Icccl/thrust -Icccl/libcudacxx/include -Icccl/cub main.cu -o main
```
> **Note**
> Use `-I` and not `-isystem` to avoid collisions with the CCCL headers implicitly included by `nvcc` from the CUDA Toolkit. All CCCL headers use `#pragma system_header` to ensure warnings will still be silenced as if using `-isystem`, see https://github.com/NVIDIA/cccl/issues/527 for more information.
##### Installation
The default CMake options generate only installation rules, so the familiar
`cmake . && make install` workflow just works:
```bash
git clone https://github.com/NVIDIA/cccl.git
cd cccl
cmake . -DCMAKE_INSTALL_PREFIX=/usr/local
make install
```
A convenience script is also provided:
```bash
ci/install_cccl.sh /usr/local
```
###### Advanced installation using presets
CMake presets are also available with options for including experimental
libraries:
```bash
cmake --preset install -DCMAKE_INSTALL_PREFIX=/usr/local
cmake --build --preset install --target install
```
Use the `install-unstable` preset to include experimental libraries, or
`install-unstable-only` to install only experimental libraries.
#### Conda
CCCL also provides conda packages of each release via the `conda-forge` channel:
```bash
conda config --add channels conda-forge
conda install cccl
```
This will install the latest CCCL to the conda environment's `$CONDA_PREFIX/include/` and `$CONDA_PREFIX/lib/cmake/` directories.
It is discoverable by CMake via `find_package(CCCL)` and can be used by any compilers in the conda environment.
For more information, see [this introduction to conda-forge](https://conda-forge.org/docs/user/introduction/).
If you want to use the same CCCL version that shipped with a particular CUDA Toolkit, e.g. CUDA 12.4, you can install CCCL with:
```bash
conda config --add channels conda-forge
conda install cuda-cccl cuda-version=12.4
```
The `cuda-cccl` metapackage installs the `cccl` version that shipped with the CUDA Toolkit corresponding to `cuda-version`.
If you wish to update to the latest `cccl` after installing `cuda-cccl`, uninstall `cuda-cccl` before updating `cccl`:
```bash
conda uninstall cuda-cccl
conda install -c conda-forge cccl
```
> **Note**
> There are also conda packages with names like `cuda-cccl_linux-64`.
> Those packages contain the CCCL versions shipped as part of the CUDA Toolkit, but are designed for internal use by the CUDA Toolkit.
> Install `cccl` or `cuda-cccl` instead, for compatibility with conda compilers.
> For more information, see the [cccl conda-forge recipe](https://github.com/conda-forge/cccl-feedstock/blob/main/recipe/meta.yaml).
##### CMake Integration
CCCL uses [CMake](https://cmake.org/) for all build and installation infrastructure, including tests as well as targets to link against in other CMake projects.
Therefore, CMake is the recommended way to integrate CCCL into another project.
For a complete example of how to do this using CMake Package Manager see [our basic example project](examples/basic).
Other build systems should work, but only CMake is tested.
Contributions to simplify integrating CCCL into other build systems are welcome.
### Contributors
Interested in contributing to making CCCL better? Check out our [Contributing Guide](CONTRIBUTING.md) for a comprehensive overview of everything you need to know to set up your development environment, make changes, run tests, and submit a PR.
## Platform Support
**Objective:** This section describes where users can expect CCCL to compile and run successfully.
In general, CCCL should work everywhere the CUDA Toolkit is supported, however, the devil is in the details.
The sections below describe the details of support and testing for different versions of the CUDA Toolkit, host compilers, and C++ dialects.
### CUDA Toolkit (CTK) Compatibility
**Summary:**
- The latest version of CCCL is backward compatible with the current and preceding CTK major version series
- CCCL is never forward compatible with any version of the CTK. Always use the same or newer than what is included with your CTK.
- Minor version CCCL upgrades won't break existing code, but new features may not support all CTK versions
CCCL users are encouraged to capitalize on the latest enhancements and ["live at head"](https://www.youtube.com/watch?v=tISy7EJQPzI) by always using the newest version of CCCL.
For a seamless experience, you can upgrade CCCL independently of the entire CUDA Toolkit.
This is possible because CCCL maintains backward compatibility with the latest patch release of every minor CTK release from both the current and previous major version series.
In some exceptional cases, the minimum supported minor version of the CUDA Toolkit release may need to be newer than the oldest release within its major version series.
When a new major CTK is released, we drop support for the oldest supported major version.
| CCCL Version | Supports CUDA Toolkit Version |
|--------------|------------------------------------------------|
| 2.x | 11.1 - 11.8, 12.x (only latest patch releases) |
| 3.x | 12.x, 13.x (only latest patch releases) |
[Well-behaved code](#compatibility-guidelines) using the latest CCCL should compile and run successfully with any supported CTK version.
Exceptions may occur for new features that depend on new CTK features, so those features would not work on older versions of the CTK.
Users can integrate a newer version of CCCL into an older CTK, but not the other way around.
This means an older version of CCCL is not compatible with a newer CTK.
In other words, **CCCL is never forward compatible with the CUDA Toolkit.**
The table below summarizes compatibility of the CTK and CCCL:
| CTK Version | Included CCCL Version | Desired CCCL | Supported? | Notes |
|:-----------:|:---------------------:|:--------------------:|:----------:|:--------------------------------------------------------:|
| CTK `X.Y` | CCCL `MAJOR.MINOR` | CCCL `MAJOR.MINOR+n` | ✅ | Some new features might not work |
| CTK `X.Y` | CCCL `MAJOR.MINOR` | CCCL `MAJOR+1.MINOR` | ✅ | Possible breaks; some new features might not be available|
| CTK `X.Y` | CCCL `MAJOR.MINOR` | CCCL `MAJOR+2.MINOR` | ❌ | CCCL supports only two CTK major versions |
| CTK `X.Y` | CCCL `MAJOR.MINOR` | CCCL `MAJOR.MINOR-n` | ❌ | CCCL isn't forward compatible |
| CTK `X.Y` | CCCL `MAJOR.MINOR` | CCCL `MAJOR-n.MINOR` | ❌ | CCCL isn't forward compatible |
For more information on CCCL versioning, API/ABI compatibility, and breaking changes see the [Versioning](#versioning) section below.
### Operating Systems
Unless otherwise specified, CCCL supports all the same operating systems as the CUDA Toolkit, which are documented here:
- [Linux](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements)
- [Windows](https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html#system-requirements)
### Host Compilers
Unless otherwise specified, CCCL supports the same host compilers as the latest CUDA Toolkit, which are documented here:
- [Linux](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#host-compiler-support-policy)
- [Windows](https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html#system-requirements)
For GCC on Linux, at least 7.x is required.
When using older CUDA Toolkits, we also only support the host compilers of the latest CUDA Toolkit,
but at least the most recent host compiler of any supported older CUDA Toolkit.
We may retain support of additional compilers and will accept corresponding patches from the community with reasonable fixes.
But we will not invest significant time in triaging or fixing issues for older compilers.
In the spirit of "You only support what you test", see our [CI Overview](https://github.com/NVIDIA/cccl/blob/main/docs/infrastructure/ci/references/ci_overview.rst) for more information on exactly what we test.
### GPU Architectures
CCCL supports all GPU architectures that are [supported by the *current* major CUDA Toolkit (CTK)](https://developer.nvidia.com/cuda-gpus).
To be clear, while CCCL can be compiled with both the current and previous CTK major versions, we do not test or validate architectures that were only supported in the older CTK.
Those architectures may still work — we do not intentionally break them — but they are outside our regular CI coverage. Furthermore, new features are not guaranteed to work with these architectures either.
We welcome community contributions for reasonable fixes that unblock users on these older architectures.
For example, CCCL 3.0 supports compiling with CTK 12.x and 13.x where
- CUDA Toolkit 13.x supports `>=sm_75`
- CUDA Toolkit 12.x supports `>=sm_50`
In this scenario, compiling CCCL 3.0 with CTK 12.x targeting architectures below `sm_75` may work, but those configurations are not part of our regular testing.
### C++ Dialects
- C++17
- C++20
### Testing Strategy
CCCL's testing strategy strikes a balance between testing as many configurations as possible and maintaining reasonable CI times.
For CUDA Toolkit versions, testing is done against both the oldest and the newest supported versions.
For instance, if the latest version of the CUDA Toolkit is 12.6, tests are conducted against 11.1 and 12.6.
For each CUDA version, builds are completed against all supported host compilers with all supported C++ dialects.
The testing strategy and matrix are constantly evolving.
The matrix defined in the [`ci/matrix.yaml`](ci/matrix.yaml) file is the definitive source of truth.
For more information about our CI pipeline, see [here](docs/infrastructure/ci/references/ci_overview.rst).
## Versioning
**Objective:** This section describes how CCCL is versioned, API/ABI stability guarantees, and compatibility guidelines to minimize upgrade headaches.
**Summary**
- The entirety of CCCL's API shares a common semantic version across all components
- Only the most recently released version is supported and fixes are not backported to prior releases
- API breaking changes and incrementing CCCL's major version will only coincide with a new major version release of the CUDA Toolkit
- Not all source breaking changes are considered breaking changes of the public API that warrant bumping the major version number
- Do not rely on ABI stability of entities in the `cub::` or `thrust::` namespaces
- ABI breaking changes for symbols in the `cuda::` namespace may happen at any time, but will be reflected by incrementing the ABI version which is embedded in an inline namespace for all `cuda::` symbols. Multiple ABI versions may be supported concurrently.
**Note:** Prior to merging Thrust, CUB, and libcudacxx into this repository, each library was independently versioned according to semantic versioning.
Starting with the 2.1 release, all three libraries synchronized their release versions in their separate repositories.
Moving forward, CCCL will continue to be released under a single [semantic version](https://semver.org/), with 2.2.0 being the first release from the [nvidia/cccl](www.github.com/nvidia/cccl) repository.
### Breaking Change
A Breaking Change is a change to **explicitly supported** functionality between released versions that would require a user to do work in order to upgrade to the newer version.
In the limit, [_any_ change](https://www.hyrumslaw.com/) has the potential to break someone somewhere.
As a result, not all possible source breaking changes are considered Breaking Changes to the public API that warrant bumping the major semantic version.
The sections below describe the details of breaking changes to CCCL's API and ABI.
### Application Programming Interface (API)
CCCL's public API is the entirety of the functionality _intentionally_ exposed to provide the utility of the library.
In other words, CCCL's public API goes beyond just function signatures and includes (but is not limited to):
- The location and names of headers intended for direct inclusion in user code
- The namespaces intended for direct use in user code
- The declarations and/or definitions of functions, classes, and variables located in headers and intended for direct use in user code
- The semantics of functions, classes, and variables intended for direct use in user code
Moreover, CCCL's public API does **not** include any of the following:
- Any symbol prefixed with `_` or `__`
- Any symbol whose name contains `detail` including the `detail::` namespace or a macro
- Any header file contained in a `detail/` directory or sub-directory thereof
- The header files implicitly included by any header part of the public API
In general, the goal is to avoid breaking anything in the public API.
Such changes are made only if they offer users better performance, easier-to-understand APIs, and/or more consistent APIs.
Any breaking change to the public API will require bumping CCCL's major version number.
In keeping with [CUDA Minor Version Compatibility](https://docs.nvidia.com/deploy/cuda-compatibility/#minor-version-compatibility),
API breaking changes and CCCL major version bumps will only occur coinciding with a new major version release of the CUDA Toolkit.
Anything not part of the public API may change at any time without warning.
#### API Versioning
The public API of all CCCL's components share a unified semantic version of `MAJOR.MINOR.PATCH`.
Only the most recently released version is supported.
As a rule, features and bug fixes are not backported to previously released version or branches.
The preferred method for querying the version is to use `CCCL_[MAJOR/MINOR/PATCH_]VERSION` as described below.
For backwards compatibility, the Thrust/CUB/libcudacxxx version definitions are available and will always be consistent with `CCCL_VERSION`.
Note that Thrust/CUB use a `MMMmmmpp` scheme whereas the CCCL and libcudacxx use `MMMmmmppp`.
| | CCCL | libcudacxx | Thrust | CUB |
|------------------------|----------------------------------------|-------------------------------------------|------------------------------|---------------------------|
| Header | `<cuda/version>` | `<cuda/std/version>` | `<thrust/version.h>` | `<cub/version.h>` |
| Major Version | `CCCL_MAJOR_VERSION` | `_LIBCUDACXX_CUDA_API_VERSION_MAJOR` | `THRUST_MAJOR_VERSION` | `CUB_MAJOR_VERSION` |
| Minor Version | `CCCL_MINOR_VERSION` | `_LIBCUDACXX_CUDA_API_VERSION_MINOR` | `THRUST_MINOR_VERSION` | `CUB_MINOR_VERSION` |
| Patch/Subminor Version | `CCCL_PATCH_VERSION` | `_LIBCUDACXX_CUDA_API_VERSION_PATCH` | `THRUST_SUBMINOR_VERSION` | `CUB_SUBMINOR_VERSION` |
| Concatenated Version | `CCCL_VERSION (MMMmmmppp)` | `_LIBCUDACXX_CUDA_API_VERSION (MMMmmmppp)`| `THRUST_VERSION (MMMmmmpp)` | `CUB_VERSION (MMMmmmpp)` |
### Application Binary Interface (ABI)
The Application Binary Interface (ABI) is a set of rules for:
- How a library's components are represented in machine code
- How those components interact across different translation units
A library's ABI includes, but is not limited to:
- The mangled names of functions and types
- The size and alignment of objects and types
- The semantics of the bytes in the binary representation of an object
An **ABI Breaking Change** is any change that results in a change to the ABI of a function or type in the public API.
For example, adding a new data member to a struct is an ABI Breaking Change as it changes the size of the type.
In CCCL, the guarantees about ABI are as follows:
- Symbols in the `thrust::` and `cub::` namespaces may break ABI at any time without warning.
- The ABI of `thrust::` and `cub::` [symbols includes the CUDA architectures used for compilation](https://nvidia.github.io/cccl/cub/developer_overview.html#symbols-visibility). Therefore, a `thrust::` or `cub::` symbol may have a different ABI if:
- compiled with different architectures
- compiled as a CUDA source file (`-x cu`) vs C++ source (`-x cpp`)
- Symbols in the `cuda::` namespace may also break ABI at any time. However, `cuda::` symbols embed an ABI version number that is incremented whenever an ABI break occurs. Multiple ABI versions may be supported concurrently, and therefore users have the option to revert to a prior ABI version. For more information, see [here](libcudacxx/docs/releases/versioning.md).
**Who should care about ABI?**
In general, CCCL users only need to worry about ABI issues when building or using a binary artifact (like a shared library) whose API directly or indirectly includes types provided by CCCL.
For example, consider if `libA.so` was built using CCCL version `X` and its public API includes a function like:
```c++
void foo(cuda::std::optional<int>);
```
If another library, `libB.so`, is compiled using CCCL version `Y` and uses `foo` from `libA.so`, then this can fail if there was an ABI break between version `X` and `Y`.
Unlike with API breaking changes, ABI breaks usually do not require code changes and only require recompiling everything to use the same ABI version.
To learn more about ABI and why it is important, see [What is ABI, and What Should C++ Do About It?](https://wg21.link/P2028R0).
### Compatibility Guidelines
As mentioned above, not all possible source breaking changes constitute a Breaking Change that would require incrementing CCCL's API major version number.
Users are encouraged to adhere to the following guidelines in order to minimize the risk of disruptions from accidentally depending on parts of CCCL that are not part of the public API:
- Do not add any declarations to, or specialize any template from, the `thrust::`, `cub::`, `nv::`, or `cuda::` namespaces unless an exception is noted for a specific symbol, e.g., specializing `cuda::std::iterator_traits`
- **Rationale**: This would cause conflicts if a symbol or specialization is added with the same name.
- Do not take the address of any API in the `thrust::`, `cub::`, `cuda::`, or `nv::` namespaces.
- **Rationale**: This would prevent adding overloads of these APIs.
- Do not forward declare any API in the `thrust::`, `cub::`, `cuda::`, or `nv::` namespaces.
- **Rationale**: This would prevent adding overloads of these APIs.
- Do not directly reference any symbol prefixed with `_`, `__`, or with `detail` anywhere in its name including a `detail::` namespace or macro
- **Rationale**: These symbols are for internal use only and may change at any time without warning.
- Include what you use. For every CCCL symbol that you use, directly `#include` the header file that declares that symbol. In other words, do not rely on headers implicitly included by other headers.
- **Rationale**: Internal includes may change at any time.
Portions of this section were inspired by [Abseil's Compatibility Guidelines](https://abseil.io/about/compatibility).
## Deprecation Policy
We will do our best to notify users prior to making any breaking changes to the public API, ABI, or modifying the supported platforms and compilers.
As appropriate, deprecations will come in the form of programmatic warnings which can be disabled.
The deprecation period will depend on the impact of the change, but will usually last at least 2 minor version releases.
## Mapping to CTK Versions
| CCCL version | CTK version |
|--------------|-------------|
| 3.2 | 13.2 |
| 3.1 | 13.1 |
| 3.0 | 13.0 |
| 2.8 | 12.9 |
| 2.7 | 12.8 |
| 2.5 | 12.6 |
| 2.4 | 12.5 |
| 2.3 | 12.4 |
Test yourself: https://cuda.godbolt.org/z/K818M4Y9f
CTKs before 12.4 shipped Thrust, CUB and libcudacxx as individual libraries.
| Thrust/CUB/libcudacxx version | CTK version |
|-------------------------------|-------------|
| 2.2 | 12.3 |
| 2.1 | 12.2 |
| 2.0/2.0/1.9 | 12.1 |
| 2.0/2.0/1.9 | 12.0 |
## CI Pipeline Overview
For a detailed overview of the CI pipeline, see [CI overview](docs/infrastructure/ci/references/ci_overview.rst).
## Related Projects
Projects that are related to CCCL's mission to make CUDA more delightful:
- [cuCollections](https://github.com/NVIDIA/cuCollections) - GPU accelerated data structures like hash tables
- [NVBench](https://github.com/NVIDIA/nvbench) - Benchmarking library tailored for CUDA applications
- [stdexec](https://github.com/nvidia/stdexec) - Reference implementation for Senders asynchronous programming model
## Projects Using CCCL
Does your project use CCCL? [Open a PR to add your project to this list!](https://github.com/NVIDIA/cccl/edit/main/README.md)
- [AmgX](https://github.com/NVIDIA/AMGX) - Multi-grid linear solver library
- [ColossalAI](https://github.com/hpcaitech/ColossalAI) - Tools for writing distributed deep learning models
- [cuDF](https://github.com/rapidsai/cudf) - Algorithms and file readers for ETL data analytics
- [cuGraph](https://github.com/rapidsai/cugraph) - Algorithms for graph analytics
- [cuML](https://github.com/rapidsai/cuml) - Machine learning algorithms and primitives
- [cuOpt](https://github.com/NVIDIA/cuopt) - Accelerated decision optimization
- [CuPy](https://cupy.dev) - NumPy & SciPy for GPU
- [cuSOLVER](https://developer.nvidia.com/cusolver) - Dense and sparse linear solvers
- [CUSP](https://github.com/cusplibrary/cusplibrary) - Sparse matrix operations, iterative methods, and algebraic multigrid
- [cuVS](https://github.com/rapidsai/cuvs) - Approximate clustering and vector search
- [GooFit](https://github.com/GooFit/GooFit) - Library for maximum-likelihood fits
- [HeavyDB](https://github.com/heavyai/heavydb) - SQL database engine
- [HOOMD](https://github.com/glotzerlab/hoomd-blue) - Monte Carlo and molecular dynamics simulations
- [HugeCTR](https://github.com/NVIDIA-Merlin/HugeCTR) - GPU-accelerated recommender framework
- [Hydra](https://github.com/MultithreadCorner/Hydra) - High-energy Physics Data Analysis
- [Hypre](https://github.com/hypre-space/hypre) - Multigrid linear solvers
- [LightSeq](https://github.com/bytedance/lightseq) - Training and inference for sequence processing and generation
- [MatX](https://github.com/NVIDIA/matx) - Numerical computing library using expression templates to provide efficient, Python-like syntax
- [Parrot](https://github.com/NVlabs/parrot) - Array fusion GPU library
- [PyTorch](https://github.com/pytorch/pytorch) - Tensor and neural network computations
- [Qiskit](https://github.com/Qiskit/qiskit-aer) - High performance simulator for quantum circuits
- [QUDA](https://github.com/lattice/quda) - Lattice quantum chromodynamics (QCD) computations
- [RAFT](https://github.com/rapidsai/raft) - Algorithms and primitives for machine learning
- [SGLang](https://github.com/sgl-project/sglang) - LLM serving framework
- [TensorFlow](https://github.com/tensorflow/tensorflow) - End-to-end platform for machine learning
- [TensorRT](https://github.com/NVIDIA/TensorRT) - Deep learning inference
- [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) - Optimized LLM inference
- [tsne-cuda](https://github.com/CannyLab/tsne-cuda) - Stochastic Neighborhood Embedding library
- [Visualization Toolkit (VTK)](https://gitlab.kitware.com/vtk/vtk) - Rendering and visualization library
- [vLLM](https://github.com/vllm-project/vllm) - LLM inference and serving
- [XGBoost](https://github.com/dmlc/xgboost) - Gradient boosting machine learning algorithms

59
cccl_upstream/benchmarks/cmake/CCCLBenchmarkRegistry.cmake Normal file
View File

@@ -0,0 +1,59 @@
cccl_get_cudatoolkit()
set(cccl_revision "")
find_package(Git)
if (GIT_FOUND)
execute_process(
COMMAND ${GIT_EXECUTABLE} describe
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
OUTPUT_VARIABLE cccl_revision
OUTPUT_STRIP_TRAILING_WHITESPACE
)
if (cccl_revision STREQUAL "")
# Sometimes, there is no tag (shallow copy)
execute_process(
COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
OUTPUT_VARIABLE cccl_revision
OUTPUT_STRIP_TRAILING_WHITESPACE
)
endif()
endif()
# Sometimes this script is used outside of a Git repository.
# In this case, we read the revision from cccl/cccl_version instead.
if ("${cccl_revision}" STREQUAL "")
file(READ "${CMAKE_SOURCE_DIR}/cccl_version" cccl_revision)
string(STRIP "${cccl_revision}" cccl_revision)
string(REPLACE "\n" "" cccl_revision "${cccl_revision}")
endif()
message(STATUS "Git revision: ${cccl_revision}")
function(get_meta_path meta_path)
set(meta_path "${CMAKE_BINARY_DIR}/cccl_meta_bench.csv" PARENT_SCOPE)
endfunction()
function(create_benchmark_registry)
get_meta_path(meta_path)
set(ctk_version "${CUDAToolkit_VERSION}")
message(STATUS "CTK version: ${ctk_version}")
file(REMOVE "${meta_path}")
file(APPEND "${meta_path}" "ctk_version,${ctk_version}\n")
file(APPEND "${meta_path}" "cccl_revision,${cccl_revision}\n")
endfunction()
function(register_cccl_tuning bench_name ranges)
get_meta_path(meta_path)
if ("${ranges}" STREQUAL "")
file(APPEND "${meta_path}" "${bench_name}\n")
else()
file(APPEND "${meta_path}" "${bench_name},${ranges}\n")
endif()
endfunction()
function(register_cccl_benchmark bench_name)
register_cccl_tuning("${bench_name}" "")
endfunction()

4
cccl_upstream/benchmarks/scripts/.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
__pycache__/
*.pyc
*.pyo
*.pyd

909
cccl_upstream/benchmarks/scripts/analyze.py Executable file
View File

@@ -0,0 +1,909 @@
#!/usr/bin/env python3
import argparse
import functools
import itertools
import json
import math
import os
import re
import cccl
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from scipy.stats import mannwhitneyu
from scipy.stats.mstats import hdquantiles
pd.options.display.max_colwidth = 100
default_colors = plt.rcParams["axes.prop_cycle"].by_key()["color"]
color_cycle = itertools.cycle(default_colors)
color_map = {}
precision = 0.01
sensitivity = 0.5
def get_bench_columns():
return ["variant", "elapsed", "center", "samples", "bw"]
def get_extended_bench_columns():
return get_bench_columns() + ["speedup", "base_samples"]
def compute_speedup(df):
bench_columns = get_bench_columns()
workload_columns = [col for col in df.columns if col not in bench_columns]
base_df = (
df[df["variant"] == "base"]
.drop(columns=["variant"])
.rename(columns={"center": "base_center", "samples": "base_samples"})
)
base_df.drop(columns=["elapsed", "bw"], inplace=True)
merged_df = df.merge(
base_df, on=[col for col in df.columns if col in workload_columns]
)
merged_df["speedup"] = merged_df["base_center"] / merged_df["center"]
merged_df = merged_df.drop(columns=["base_center"])
return merged_df
def get_ct_axes(df):
ct_axes = []
for col in df.columns:
if "{ct}" in col:
ct_axes.append(col)
return ct_axes
def get_rt_axes(df):
rt_axes = []
excluded_columns = get_ct_axes(df) + get_extended_bench_columns()
for col in df.columns:
if col not in excluded_columns:
rt_axes.append(col)
return rt_axes
def ct_space(df):
ct_axes = get_ct_axes(df)
unique_ct_combinations = []
for _, row in df[ct_axes].drop_duplicates().iterrows():
unique_ct_combinations.append({})
for col in ct_axes:
unique_ct_combinations[-1][col] = row[col]
return unique_ct_combinations
def extract_case(df, ct_point):
tuning_df_loc = None
for ct_axis in ct_point:
if tuning_df_loc is None:
tuning_df_loc = df[ct_axis] == ct_point[ct_axis]
else:
tuning_df_loc = tuning_df_loc & (df[ct_axis] == ct_point[ct_axis])
tuning_df = df.loc[tuning_df_loc].copy()
for ct_axis in ct_point:
tuning_df.drop(columns=[ct_axis], inplace=True)
return tuning_df
def extract_rt_axes_values(df):
rt_axes = get_rt_axes(df)
rt_axes_values = {}
for rt_axis in rt_axes:
rt_axes_values[rt_axis] = list(df[rt_axis].unique())
return rt_axes_values
def extract_rt_space(df):
rt_axes = get_rt_axes(df)
rt_axes_values = []
for rt_axis in rt_axes:
values = df[rt_axis].unique()
rt_axes_values.append(["{}={}".format(rt_axis, v) for v in values])
return list(itertools.product(*rt_axes_values))
def filter_variants(df, group):
rt_axes = get_rt_axes(df)
unique_combinations = set(df[rt_axes].drop_duplicates().itertuples(index=False))
group_combinations = set(group[rt_axes].drop_duplicates().itertuples(index=False))
has_all_combinations = group_combinations == unique_combinations
return has_all_combinations
def extract_complete_variants(df):
return df.groupby("variant").filter(functools.partial(filter_variants, df))
def compute_workload_score(rt_axes_values, rt_axes_ids, weights, row):
rt_workload = []
for rt_axis in rt_axes_values:
rt_workload.append("{}={}".format(rt_axis, row[rt_axis]))
weight = cccl.bench.get_workload_weight(
rt_workload, rt_axes_values, rt_axes_ids, weights
)
return row["speedup"] * weight
def compute_variant_score(rt_axes_values, rt_axes_ids, weight_matrix, group):
workload_score_closure = functools.partial(
compute_workload_score, rt_axes_values, rt_axes_ids, weight_matrix
)
score_sum = group.apply(workload_score_closure, axis=1).sum()
return score_sum
def extract_scores(dfs):
rt_axes_values = {}
for subbench in dfs:
rt_axes_values[subbench] = extract_rt_axes_values(dfs[subbench])
rt_axes_ids = cccl.bench.compute_axes_ids(rt_axes_values)
weights = cccl.bench.compute_weight_matrices(rt_axes_values, rt_axes_ids)
score_dfs = []
for subbench in dfs:
score_closure = functools.partial(
compute_variant_score,
rt_axes_values[subbench],
rt_axes_ids[subbench],
weights[subbench],
)
grouped = dfs[subbench].groupby("variant")
scores = grouped.apply(score_closure, include_groups=False).reset_index()
scores.columns = ["variant", "score"]
stat = grouped.agg(
mins=("speedup", "min"), means=("speedup", "mean"), maxs=("speedup", "max")
)
scores = pd.merge(scores, stat, on="variant")
score_dfs.append(scores)
score_df = pd.concat(score_dfs)
result = (
score_df.groupby("variant")
.agg({"score": "sum", "mins": "min", "means": "mean", "maxs": "max"})
.reset_index()
)
return result.sort_values(by=["score"], ascending=False)
def distributions_are_different(alpha, row):
ref_samples = row["base_samples"]
cmp_samples = row["samples"]
# H0: the distributions are not different
# H1: the distribution are different
_, p = mannwhitneyu(ref_samples, cmp_samples)
# Reject H0
return p < alpha
def remove_matching_distributions(alpha, df):
closure = functools.partial(distributions_are_different, alpha)
return df[df.apply(closure, axis=1)]
def get_filenames_map(arr):
if not arr:
return []
prefix = arr[0]
for string in arr:
while not string.startswith(prefix):
prefix = prefix[:-1]
if not prefix:
break
return {string: string[len(prefix) :] for string in arr}
def is_finite(x):
if isinstance(x, float):
return x != np.inf and x != -np.inf
return True
def iterate_case_dfs(args, callable):
storages = {}
algnames = set()
filenames_map = get_filenames_map(args.files)
for file in args.files:
storage = cccl.bench.SQLiteStorage(file)
algnames.update(storage.algnames())
storages[filenames_map[file]] = storage
pattern = re.compile(args.R)
exact_values = {}
if args.args:
for value in args.args:
name, val = value.split("=")
exact_values[name] = val
for algname in algnames:
if not pattern.match(algname):
continue
case_dfs = {}
for file in storages:
storage = storages[file]
for subbench in storage.subbenches(algname):
df = storage.alg_to_df(algname, subbench)
df = df.map(lambda x: x if is_finite(x) else np.nan)
df = df.dropna(subset=["center"], how="all")
for _, row in df[["ctk", "cccl"]].drop_duplicates().iterrows():
ctk_version = row["ctk"]
cccl_version = row["cccl"]
ctk_cub_df = df[
(df["ctk"] == ctk_version) & (df["cccl"] == cccl_version)
]
for gpu in ctk_cub_df["gpu"].unique():
target_df = ctk_cub_df[ctk_cub_df["gpu"] == gpu]
target_df = target_df.drop(columns=["ctk", "cccl", "gpu"])
target_df = compute_speedup(target_df)
for key in exact_values:
if key in target_df.columns:
target_df = target_df[
target_df[key] == exact_values[key]
]
for ct_point in ct_space(target_df):
point_str = ", ".join(
["{}={}".format(k, ct_point[k]) for k in ct_point]
)
case_df = extract_complete_variants(
extract_case(target_df, ct_point)
)
case_df["variant"] = case_df["variant"].astype(
str
) + " ({})".format(file)
if point_str not in case_dfs:
case_dfs[point_str] = {}
if subbench not in case_dfs[point_str]:
case_dfs[point_str][subbench] = case_df
else:
case_dfs[point_str][subbench] = pd.concat(
[case_dfs[point_str][subbench], case_df]
)
for point_str in case_dfs:
callable(algname, point_str, case_dfs[point_str])
def case_top(alpha, N, algname, ct_point_name, case_dfs):
print("{}[{}]:".format(algname, ct_point_name))
if alpha < 1.0:
for subbench in case_dfs:
case_dfs[subbench] = remove_matching_distributions(
alpha, case_dfs[subbench]
)
for subbench in case_dfs:
case_dfs[subbench] = extract_complete_variants(case_dfs[subbench])
with pd.option_context("display.max_rows", None):
print(extract_scores(case_dfs).head(N))
def top(args):
iterate_case_dfs(args, functools.partial(case_top, args.alpha, args.top))
def case_coverage(algname, ct_point_name, case_dfs):
num_variants = cccl.bench.Config().variant_space_size(algname)
min_coverage = 100.0
for subbench in case_dfs:
num_covered_variants = len(case_dfs[subbench]["variant"].unique())
coverage = (num_covered_variants / num_variants) * 100
min_coverage = min(min_coverage, coverage)
case_str = "{}[{}]".format(algname, ct_point_name)
print(
"{} coverage: {} / {} ({:.4f}%)".format(
case_str, num_covered_variants, num_variants, min_coverage
)
)
def coverage(args):
iterate_case_dfs(args, case_coverage)
def parallel_coordinates_plot(df, title):
# Parallel coordinates plot adaptation of https://stackoverflow.com/a/69411450
import matplotlib.cm as cm
import matplotlib.patches as patches
from matplotlib.path import Path
# Variables (the first variable must be categoric):
my_vars = df.columns.tolist()
df_plot = df[my_vars]
df_plot = df_plot.dropna()
df_plot = df_plot.reset_index(drop=True)
# Convert to numeric matrix:
ym = []
dics_vars = []
for v, var in enumerate(my_vars):
if df_plot[var].dtype.kind not in ["i", "u", "f"]:
dic_var = dict([(val, c) for c, val in enumerate(df_plot[var].unique())])
dics_vars += [dic_var]
ym += [[dic_var[i] for i in df_plot[var].tolist()]]
else:
ym += [df_plot[var].tolist()]
ym = np.array(ym).T
# Padding:
ymins = ym.min(axis=0)
ymaxs = ym.max(axis=0)
dys = ymaxs - ymins
ymins -= dys * 0.05
ymaxs += dys * 0.05
dys = ymaxs - ymins
# Adjust to the main axis:
zs = np.zeros_like(ym)
zs[:, 0] = ym[:, 0]
zs[:, 1:] = (ym[:, 1:] - ymins[1:]) / dys[1:] * dys[0] + ymins[0]
# Plot:
fig, host_ax = plt.subplots(figsize=(20, 10), tight_layout=True)
# Make the axes:
axes = [host_ax] + [host_ax.twinx() for i in range(ym.shape[1] - 1)]
dic_count = 0
for i, ax in enumerate(axes):
ax.set_ylim(bottom=ymins[i], top=ymaxs[i])
ax.spines.top.set_visible(False)
ax.spines.bottom.set_visible(False)
ax.ticklabel_format(style="plain")
if ax != host_ax:
ax.spines.left.set_visible(False)
ax.yaxis.set_ticks_position("right")
ax.spines.right.set_position(("axes", i / (ym.shape[1] - 1)))
if df_plot.iloc[:, i].dtype.kind not in ["i", "u", "f"]:
dic_var_i = dics_vars[dic_count]
ax.set_yticks(range(len(dic_var_i)))
if i == 0:
ax.set_yticklabels([])
else:
ax.set_yticklabels([key_val for key_val in dics_vars[dic_count].keys()])
dic_count += 1
host_ax.set_xlim(left=0, right=ym.shape[1] - 1)
host_ax.set_xticks(range(ym.shape[1]))
host_ax.set_xticklabels(my_vars, fontsize=14)
host_ax.tick_params(axis="x", which="major", pad=7)
# Color map:
colormap = cm.get_cmap("turbo")
# Normalize speedups:
df["speedup_normalized"] = (df["speedup"] - df["speedup"].min()) / (
df["speedup"].max() - df["speedup"].min()
)
# Make the curves:
host_ax.spines.right.set_visible(False)
host_ax.xaxis.tick_top()
for j in range(ym.shape[0]):
verts = list(
zip(
[
x
for x in np.linspace(0, len(ym) - 1, len(ym) * 3 - 2, endpoint=True)
],
np.repeat(zs[j, :], 3)[1:-1],
)
)
codes = [Path.MOVETO] + [Path.CURVE4 for _ in range(len(verts) - 1)]
path = Path(verts, codes)
color_first_cat_var = colormap(df.loc[j, "speedup_normalized"])
patch = patches.PathPatch(
path, facecolor="none", lw=2, alpha=0.05, edgecolor=color_first_cat_var
)
host_ax.add_patch(patch)
host_ax.set_title(title)
plt.show()
def case_coverage_plot(algname, ct_point_name, case_dfs):
data_list = []
for subbench in case_dfs:
for _, row_description in case_dfs[subbench].iterrows():
variant = row_description["variant"]
speedup = row_description["speedup"]
if variant.startswith("base"):
continue
varname, _ = variant.split(" ")
params = varname.split(".")
data_dict = {"variant": variant}
for param in params:
print(variant)
name, val = param.split("_")
data_dict[name] = int(val)
data_dict["speedup"] = speedup
# data_dict['variant'] = variant
data_list.append(data_dict)
df = pd.DataFrame(data_list)
parallel_coordinates_plot(df, "{} ({})".format(algname, ct_point_name))
def coverage_plot(args):
iterate_case_dfs(args, case_coverage_plot)
def case_pair_plot(algname, ct_point_name, case_dfs):
import seaborn as sns
data_list = []
for subbench in case_dfs:
for _, row_description in case_dfs[subbench].iterrows():
variant = row_description["variant"]
speedup = row_description["speedup"]
if variant.startswith("base"):
continue
varname, _ = variant.split(" ")
params = varname.split(".")
data_dict = {}
for param in params:
print(variant)
name, val = param.split("_")
data_dict[name] = int(val)
data_dict["speedup"] = speedup
data_list.append(data_dict)
df = pd.DataFrame(data_list)
sns.pairplot(df, hue="speedup")
plt.title("{} ({})".format(algname, ct_point_name))
plt.show()
def pair_plot(args):
iterate_case_dfs(args, case_pair_plot)
def qrde_hd(samples):
"""
Computes quantile-respectful density estimation based on the Harrell-Davis
quantile estimator. The implementation is based on the following post:
https://aakinshin.net/posts/qrde-hd by Andrey Akinshin
"""
min_sample, max_sample = min(samples), max(samples)
num_quantiles = math.ceil(1.0 / precision)
quantiles = np.linspace(precision, 1 - precision, num_quantiles - 1)
hd_quantiles = [min_sample] + list(hdquantiles(samples, quantiles)) + [max_sample]
width = [hd_quantiles[idx + 1] - hd_quantiles[idx] for idx in range(num_quantiles)]
p = 1.0 / precision
height = [1.0 / (p * w) for w in width]
return width, height
def hd_quantiles(samples):
min_sample, max_sample = min(samples), max(samples)
num_quantiles = math.ceil(1.0 / precision)
quantiles = np.linspace(precision, 1 - precision, num_quantiles - 1)
hd_quantiles = [min_sample] + list(hdquantiles(samples, quantiles)) + [max_sample]
return hd_quantiles
def extract_peaks(pdf):
peaks = []
for i in range(1, len(pdf) - 1):
if pdf[i - 1] < pdf[i] > pdf[i + 1]:
peaks.append(i)
return peaks
def extract_modes(samples):
"""
Extract modes from the given samples based on the lowland algorithm:
https://aakinshin.net/posts/lowland-multimodality-detection/ by Andrey Akinshin
Implementation is based on the https://github.com/AndreyAkinshin/perfolizer
LowlandModalityDetector class.
"""
mode_ids = []
widths, heights = hd_displot(samples)
peak_ids = extract_peaks(heights)
bin_area = 1.0 / len(heights)
x = min(samples)
peak_xs = []
peak_ys = []
bin_lower = [x]
for idx in range(len(heights)):
if idx in peak_ids:
peak_ys.append(heights[idx])
peak_xs.append(x + widths[idx] / 2)
x += widths[idx]
bin_lower.append(x)
def lowland_between(mode_candidate, left_peak, right_peak):
left, right = left_peak, right_peak
min_height = min(heights[left_peak], heights[right_peak])
while left < right and heights[left] > min_height:
left += 1
while left < right and heights[right] > min_height:
right -= 1
width = bin_lower[right + 1] - bin_lower[left]
total_area = width * min_height
total_bin_area = (right - left + 1) * bin_area
if total_bin_area / total_area < sensitivity:
mode_ids.append(mode_candidate)
return True
return False
previousPeaks = [peak_ids[0]]
for i in range(1, len(peak_ids)):
currentPeak = peak_ids[i]
while previousPeaks and heights[previousPeaks[-1]] < heights[currentPeak]:
if lowland_between(previousPeaks[0], previousPeaks[-1], currentPeak):
previousPeaks = []
else:
previousPeaks.pop()
if previousPeaks and heights[previousPeaks[-1]] > heights[currentPeak]:
if lowland_between(previousPeaks[0], previousPeaks[-1], currentPeak):
previousPeaks = []
previousPeaks.append(currentPeak)
mode_ids.append(previousPeaks[0])
return mode_ids
def hd_displot(samples, label, ax):
if label not in color_map:
color_map[label] = next(color_cycle)
color = color_map[label]
widths, heights = qrde_hd(samples)
mode_ids = extract_modes(samples)
min_sample, max_sample = min(samples), max(samples)
xs = [min_sample]
ys = [0]
peak_xs = []
peak_ys = []
x = min(samples)
for idx in range(len(widths)):
xs.append(x + widths[idx] / 2)
ys.append(heights[idx])
if idx in mode_ids:
peak_ys.append(heights[idx])
peak_xs.append(x + widths[idx] / 2)
x += widths[idx]
xs = xs + [max_sample]
ys = ys + [0]
ax.fill_between(xs, ys, 0, alpha=0.4, color=color)
quartiles_of_interest = [0.25, 0.5, 0.75]
for quartile in quartiles_of_interest:
bin = int(quartile / precision) + 1
ax.plot([xs[bin], xs[bin]], [0, ys[bin]], color=color)
ax.plot(xs, ys, label=label, color=color)
ax.plot(peak_xs, peak_ys, "o", color=color)
ax.legend()
def displot(data, ax):
for variant in data:
hd_displot(data[variant], variant, ax)
def variant_ratio(data, variant, ax):
if variant not in color_map:
color_map[variant] = next(color_cycle)
color = color_map[variant]
variant_samples = data[variant]
base_samples = data["base"]
variant_widths = hd_quantiles(variant_samples)
base_widths = hd_quantiles(base_samples)
quantiles = []
ratios = []
base_x = min(base_samples)
variant_x = min(variant_samples)
for i in range(1, len(variant_widths) - 1):
base_x += base_widths[i] / 2
variant_x += variant_widths[i] / 2
quantiles.append(i * precision)
ratios.append(base_x / variant_x)
ax.plot(quantiles, ratios, label=variant, color=color)
ax.axhline(1, color="red", alpha=0.7)
ax.legend()
ax.tick_params(axis="both", direction="in", pad=-22)
def ratio(data, ax):
for variant in data:
if variant != "base":
variant_ratio(data, variant, ax)
def case_variants(pattern, mode, algname, ct_point_name, case_dfs):
for subbench in case_dfs:
case_df = case_dfs[subbench]
title = "{}[{}]:".format(algname + "/" + subbench, ct_point_name)
df = case_df[case_df["variant"].str.contains(pattern, regex=True)].reset_index(
drop=True
)
rt_axes = get_rt_axes(df)
rt_axes_values = extract_rt_axes_values(df)
vertical_axis_name = rt_axes[0]
if "Elements{io}[pow2]" in rt_axes:
vertical_axis_name = "Elements{io}[pow2]"
horizontal_axes = rt_axes
horizontal_axes.remove(vertical_axis_name)
vertical_axis_values = rt_axes_values[vertical_axis_name]
vertical_axis_ids = {}
for idx, val in enumerate(vertical_axis_values):
vertical_axis_ids[val] = idx
def extract_horizontal_space(df):
values = []
for rt_axis in horizontal_axes:
values.append(
["{}={}".format(rt_axis, v) for v in df[rt_axis].unique()]
)
return list(itertools.product(*values))
if len(horizontal_axes) > 0:
idx = 0
horizontal_axis_ids = {}
for point in extract_horizontal_space(df):
horizontal_axis_ids[" / ".join(point)] = idx
idx = idx + 1
num_rows = len(vertical_axis_ids)
num_cols = max(1, len(extract_horizontal_space(df)))
if num_rows == 0:
return
fig, axes = plt.subplots(
nrows=num_rows, ncols=num_cols, gridspec_kw={"wspace": 0, "hspace": 0}
)
for _, vertical_row_description in (
df[[vertical_axis_name]].drop_duplicates().iterrows()
):
vertical_val = vertical_row_description[vertical_axis_name]
vertical_id = vertical_axis_ids[vertical_val]
vertical_name = "{}={}".format(vertical_axis_name, vertical_val)
vertical_df = df[df[vertical_axis_name] == vertical_val]
for _, horizontal_row_description in (
vertical_df[horizontal_axes].drop_duplicates().iterrows()
):
horizontal_df = vertical_df
for axis in horizontal_axes:
horizontal_df = horizontal_df[
horizontal_df[axis] == horizontal_row_description[axis]
]
horizontal_id = 0
if len(horizontal_axes) > 0:
horizontal_point = []
for rt_axis in horizontal_axes:
horizontal_point.append(
"{}={}".format(rt_axis, horizontal_row_description[rt_axis])
)
horizontal_name = " / ".join(horizontal_point)
horizontal_id = horizontal_axis_ids[horizontal_name]
ax = axes[vertical_id, horizontal_id]
else:
ax = axes[vertical_id]
ax.set_ylabel(vertical_name)
data = {}
for _, variant in (
horizontal_df[["variant"]].drop_duplicates().iterrows()
):
variant_name = variant["variant"]
if "base" not in data:
data["base"] = horizontal_df[
horizontal_df["variant"] == variant_name
].iloc[0]["base_samples"]
data[variant_name] = horizontal_df[
horizontal_df["variant"] == variant_name
].iloc[0]["samples"]
if mode == "pdf":
# sns.histplot(data=data, ax=ax, kde=True)
displot(data, ax)
else:
ratio(data, ax)
if len(horizontal_axes) > 0:
ax = axes[vertical_id, horizontal_id]
if vertical_id == (num_rows - 1):
ax.set_xlabel(horizontal_name)
if horizontal_id == 0:
ax.set_ylabel(vertical_name)
else:
ax.set_ylabel("")
for ax in axes.flat:
ax.set_xticklabels([])
fig.suptitle(title)
plt.tight_layout()
plt.show()
def variants(args, mode):
pattern = (
re.compile(args.variants_pdf)
if mode == "pdf"
else re.compile(args.variants_ratio)
)
iterate_case_dfs(args, functools.partial(case_variants, pattern, mode))
def file_exists(value):
if not os.path.isfile(value):
raise argparse.ArgumentTypeError(f"The file '{value}' does not exist.")
return value
def case_offload(algname, ct_point_name, case_dfs):
for subbench in case_dfs:
df = case_dfs[subbench]
for rt_point in extract_rt_space(df):
point_df = df
for rt_kv in rt_point:
key, value = rt_kv.split("=")
point_df = point_df[point_df[key] == value]
point_name = ct_point_name + " " + " ".join(rt_point)
point_name = point_name.replace(",", "")
bench_name = "{}.{}-{}".format(algname, subbench, point_name)
bench_name = bench_name.replace(" ", "___")
bench_name = "".join(c if c.isalnum() else "_" for c in bench_name)
with open(bench_name + ".json", "w") as f:
obj = json.loads(point_df.to_json(orient="records"))
json.dump(obj, f, indent=2)
def offload(args):
iterate_case_dfs(args, case_offload)
def parse_arguments():
parser = argparse.ArgumentParser(description="Analyze benchmark results.")
parser.add_argument(
"-R", type=str, default=".*", help="Regex for benchmarks selection."
)
parser.add_argument(
"--list-benches",
action=argparse.BooleanOptionalAction,
help="Show available benchmarks.",
)
parser.add_argument(
"--coverage",
action=argparse.BooleanOptionalAction,
help="Show variant space coverage.",
)
parser.add_argument(
"--coverage-plot",
action=argparse.BooleanOptionalAction,
help="Plot variant space coverage.",
)
parser.add_argument(
"--pair-plot", action=argparse.BooleanOptionalAction, help="Pair plot."
)
parser.add_argument(
"--top",
default=7,
type=int,
action="store",
nargs="?",
help="Show top N variants with highest score.",
)
parser.add_argument(
"files", type=file_exists, nargs="+", help="At least one file is required."
)
parser.add_argument("--alpha", default=1.0, type=float)
parser.add_argument("--variants-pdf", type=str, help="Show matching variants data.")
parser.add_argument(
"--variants-ratio", type=str, help="Show matching variants data."
)
parser.add_argument(
"-a",
"--args",
action="append",
type=str,
help="Parameter in the format `Param=Value`.",
)
parser.add_argument(
"-o", "--offload", action=argparse.BooleanOptionalAction, help="Offload samples"
)
return parser.parse_args()
def main():
args = parse_arguments()
if args.list_benches:
cccl.bench.list_benches()
return
if args.coverage:
coverage(args)
return
if args.coverage_plot:
coverage_plot(args)
return
if args.pair_plot:
pair_plot(args)
return
if args.variants_pdf:
variants(args, "pdf")
return
if args.variants_ratio:
variants(args, "ratio")
return
if args.offload:
offload(args)
return
top(args)
if __name__ == "__main__":
main()

3
cccl_upstream/benchmarks/scripts/cccl/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
from . import bench
__all__ = ["bench"]

6
cccl_upstream/benchmarks/scripts/cccl/bench/__init__.py Normal file
View File

@@ -0,0 +1,6 @@
from .bench import Bench # noqa: F401
from .cmake import CMake # noqa: F401
from .config import * # noqa: F403
from .score import * # noqa: F403
from .search import * # noqa: F403
from .storage import * # noqa: F403

814
cccl_upstream/benchmarks/scripts/cccl/bench/bench.py Normal file
View File

@@ -0,0 +1,814 @@
import itertools
import json
import os
import signal
import subprocess
import time
import fpzip
import numpy as np
from .cmake import CMake
from .config import BasePoint, Config
from .logger import Logger
from .score import compute_axes_ids, compute_weight_matrices, get_workload_weight
from .storage import Storage, get_bench_table_name
def first_val(my_dict):
values = list(my_dict.values())
first_value = values[0]
if not all(value == first_value for value in values):
raise ValueError(
"All values in the dictionary are not equal. First value: {} All values: {}".format(
first_value, values
)
)
return first_value
class JsonCache:
_instance = None
def __new__(cls):
if cls._instance is None:
cls._instance = super().__new__(cls)
cls._instance.bench_cache = {}
cls._instance.device_cache = {}
return cls._instance
def get_jsonlist(self, algname, listname):
benchmark_bin = os.path.join(".", "bin", algname + ".base")
if not os.path.exists(benchmark_bin):
raise Exception(f"Benchmark binary not found: {benchmark_bin}")
return subprocess.check_output([benchmark_bin, f"--jsonlist-{listname}"])
def get_bench(self, algname):
if algname not in self.bench_cache:
result = self.get_jsonlist(algname, "benches")
self.bench_cache[algname] = json.loads(result)
return self.bench_cache[algname]
def get_device(self, algname):
if algname not in self.device_cache:
result = self.get_jsonlist(algname, "devices")
data = json.loads(result)
if "devices" not in data:
raise Exception(
"JSON returned from --jsonlist-devices does not contain 'devices' key"
)
devices = data["devices"]
if len(devices) != 1:
raise Exception(
"NVBench doesn't work well with multiple GPUs, use `CUDA_VISIBLE_DEVICES`"
)
self.device_cache[algname] = devices[0]
return self.device_cache[algname]
def json_benches(algname):
return JsonCache().get_bench(algname)
def create_benches_tables(conn, subbench, bench_axes):
with conn:
conn.execute("""
CREATE TABLE IF NOT EXISTS subbenches (
algorithm TEXT NOT NULL,
bench TEXT NOT NULL,
UNIQUE(algorithm, bench)
);
""")
for algorithm_name in bench_axes:
axes = bench_axes[algorithm_name]
column_names = ", ".join(['"{}"'.format(name) for name in axes])
columns = ", ".join(['"{}" TEXT'.format(name) for name in axes])
conn.execute(
"""
INSERT INTO subbenches (algorithm, bench)
VALUES (?, ?)
ON CONFLICT DO NOTHING;
""",
(algorithm_name, subbench),
)
if axes:
columns = ", " + columns
column_names = ", " + column_names
conn.execute(
"""
CREATE TABLE IF NOT EXISTS "{0}" (
ctk TEXT NOT NULL,
cccl TEXT NOT NULL,
gpu TEXT NOT NULL,
variant TEXT NOT NULL,
elapsed REAL,
center REAL,
bw REAL,
samples BLOB
{1}
, UNIQUE(ctk, cccl, gpu, variant {2})
);
""".format(
get_bench_table_name(subbench, algorithm_name),
columns,
column_names,
)
)
def read_json(filename):
with open(filename, "r") as f:
file_root = json.load(f)
return file_root
def extract_filename(summary):
summary_data = summary["data"]
value_data = next(filter(lambda v: v["name"] == "filename", summary_data))
assert value_data["type"] == "string"
return value_data["value"]
def extract_size(summary):
summary_data = summary["data"]
value_data = next(filter(lambda v: v["name"] == "size", summary_data))
assert value_data["type"] == "int64"
return int(value_data["value"])
def extract_bw(summary):
summary_data = summary["data"]
value_data = next(filter(lambda v: v["name"] == "value", summary_data))
assert value_data["type"] == "float64"
return float(value_data["value"])
def parse_samples_meta(state):
summaries = state["summaries"]
if not summaries:
return None, None
summary = next(
filter(lambda s: s["tag"] == "nv/json/bin:nv/cold/sample_times", summaries),
None,
)
if not summary:
return None, None
sample_filename = extract_filename(summary)
sample_count = extract_size(summary)
return sample_count, sample_filename
def parse_samples(state):
sample_count, samples_filename = parse_samples_meta(state)
if not sample_count or not samples_filename:
return np.array([], dtype=np.float32)
with open(samples_filename, "rb") as f:
samples = np.fromfile(f, "<f4")
samples.sort()
assert sample_count == len(samples)
return samples
def parse_bw(state):
bwutil = next(
filter(
lambda s: s["tag"] == "nv/cold/bw/global/utilization", state["summaries"]
),
None,
)
if not bwutil:
return None
return extract_bw(bwutil)
class SubBenchState:
def __init__(self, state, axes_names, axes_values):
self.samples = parse_samples(state)
self.bw = parse_bw(state)
self.point = {}
for axis in state["axis_values"]:
name = axes_names[axis["name"]]
value = axes_values[axis["name"]][axis["value"]]
self.point[name] = value
def __repr__(self):
return str(self.__dict__)
def name(self):
return " ".join(f"{k}={v}" for k, v in self.point.items())
def center(self, estimator):
return estimator(self.samples)
class SubBenchResult:
def __init__(self, bench):
axes_names = {}
axes_values = {}
for axis in bench["axes"]:
short_name = axis["name"]
full_name = get_axis_name(axis)
axes_names[short_name] = full_name
axes_values[short_name] = {}
for value in axis["values"]:
if "value" in value:
axes_values[axis["name"]][str(value["value"])] = value[
"input_string"
]
else:
axes_values[axis["name"]][value["input_string"]] = value[
"input_string"
]
self.states = []
for state in bench["states"]:
if not state["is_skipped"]:
self.states.append(SubBenchState(state, axes_names, axes_values))
def __repr__(self):
return str(self.__dict__)
def centers(self, estimator):
result = {}
for state in self.states:
result[state.name()] = state.center(estimator)
return result
class BenchResult:
def __init__(self, json_path, code, elapsed):
self.code = code
self.elapsed = elapsed
if json_path:
self.subbenches = {}
if code == 0:
for bench in read_json(json_path)["benchmarks"]:
self.subbenches[bench["name"]] = SubBenchResult(bench)
def __repr__(self):
return str(self.__dict__)
def centers(self, estimator):
result = {}
for subbench in self.subbenches:
result[subbench] = self.subbenches[subbench].centers(estimator)
return result
def device_json(algname):
return JsonCache().get_device(algname)
CCCL_BENCH_GPU_ENV = "CCCL_BENCH_GPU"
def get_gpu_name_override():
override = os.environ.get(CCCL_BENCH_GPU_ENV)
if override is not None and override.strip():
return override.strip()
return None
def get_device_name(device):
gpu_name = device["name"]
bus_width = device["global_memory_bus_width"]
sms = device["number_of_sms"]
ecc = "eccon" if device["ecc_state"] else "eccoff"
name = "{} ({}, {}, {})".format(gpu_name, bus_width, sms, ecc)
return name.replace("NVIDIA ", "")
def get_gpu_name(algname):
override = get_gpu_name_override()
if override is not None:
return override
return get_device_name(device_json(algname))
def is_ct_axis(name):
return "{ct}" in name
def state_to_rt_workload(bench, state):
rt_workload = []
for param in state.split(" "):
name, value = param.split("=")
if is_ct_axis(name):
continue
rt_workload.append("{}={}".format(name, value))
return rt_workload
def create_runs_table(conn):
with conn:
conn.execute("""
CREATE TABLE IF NOT EXISTS runs (
ctk TEXT NOT NULL,
cccl TEXT NOT NULL,
bench TEXT NOT NULL,
code TEXT NOT NULL,
elapsed REAL
);
""")
class RunsCache:
_instance = None
def __new__(cls, *args, **kwargs):
if cls._instance is None:
cls._instance = super().__new__(cls, *args, **kwargs)
create_runs_table(Storage().connection())
return cls._instance
def pull_run(self, bench):
config = Config()
ctk = config.ctk
cccl = config.cccl
conn = Storage().connection()
with conn:
query = "SELECT code, elapsed FROM runs WHERE ctk = ? AND cccl = ? AND bench = ?;"
result = conn.execute(query, (ctk, cccl, bench.label())).fetchone()
if result:
code, elapsed = result
return int(code), float(elapsed)
return result
def push_run(self, bench, code, elapsed):
config = Config()
ctk = config.ctk
cccl = config.cccl
conn = Storage().connection()
with conn:
conn.execute(
"INSERT INTO runs (ctk, cccl, bench, code, elapsed) VALUES (?, ?, ?, ?, ?);",
(ctk, cccl, bench.label(), code, elapsed),
)
class BenchCache:
_instance = None
def __new__(cls, *args, **kwargs):
if cls._instance is None:
cls._instance = super().__new__(cls, *args, **kwargs)
cls._instance.existing_tables = set()
return cls._instance
def create_table_if_not_exists(self, conn, bench):
bench_base = bench.get_base()
alg_name = bench_base.algorithm_name()
if alg_name not in self.existing_tables:
subbench_axes_names = bench_base.axes_names()
for subbench in subbench_axes_names:
create_benches_tables(
conn, subbench, {alg_name: subbench_axes_names[subbench]}
)
self.existing_tables.add(alg_name)
def push_bench_centers(self, bench, result, estimator):
config = Config()
ctk = config.ctk
cccl = config.cccl
gpu = get_gpu_name(bench.algname)
conn = Storage().connection()
self.create_table_if_not_exists(conn, bench)
centers = {}
with conn:
for subbench in result.subbenches:
centers[subbench] = {}
for state in result.subbenches[subbench].states:
table_name = get_bench_table_name(subbench, bench.algorithm_name())
columns = ""
placeholders = ""
values = []
for name in state.point:
value = state.point[name]
columns = columns + ', "{}"'.format(name)
placeholders = placeholders + ", ?"
values.append(value)
values = tuple(values)
samples = fpzip.compress(state.samples)
center = estimator(state.samples)
to_insert = (
ctk,
cccl,
gpu,
bench.variant_name(),
result.elapsed,
center,
state.bw,
samples,
) + values
query = """
INSERT INTO "{0}" (ctk, cccl, gpu, variant, elapsed, center, bw, samples {1})
VALUES (?, ?, ?, ?, ?, ?, ?, ? {2})
ON CONFLICT(ctk, cccl, gpu, variant {1}) DO NOTHING;
""".format(table_name, columns, placeholders)
conn.execute(query, to_insert)
centers[subbench][state.name()] = center
return centers
def pull_bench_centers(self, bench, ct_workload_point, rt_values):
config = Config()
ctk = config.ctk
cccl = config.cccl
gpu = get_gpu_name(bench.algname)
conn = Storage().connection()
self.create_table_if_not_exists(conn, bench)
centers = {}
with conn:
for subbench in rt_values:
centers[subbench] = {}
table_name = get_bench_table_name(subbench, bench.algorithm_name())
for rt_point in values_to_space(rt_values[subbench]):
point_map = {}
point_checks = ""
workload_point = list(ct_workload_point) + list(rt_point)
for axis in workload_point:
name, value = axis.split("=")
point_map[name] = value
point_checks = point_checks + ' AND "{}" = "{}"'.format(
name, value
)
query = """
SELECT center FROM "{0}" WHERE ctk = ? AND cccl = ? AND gpu = ? AND variant = ?{1};
""".format(table_name, point_checks)
result = conn.execute(
query, (ctk, cccl, gpu, bench.variant_name())
).fetchone()
if result is None:
return None
state_name = " ".join(f"{k}={v}" for k, v in point_map.items())
centers[subbench][state_name] = float(result[0])
return centers
def get_axis_name(axis):
name = axis["name"]
if axis["flags"]:
name = name + "[{}]".format(axis["flags"])
return name
def speedup(base, variant):
# If one of the runs failed, dict is empty
if not base or not variant:
return {}
benchmarks = set(base.keys())
if benchmarks != set(variant.keys()):
raise Exception("Benchmarks do not match.")
result = {}
for bench in benchmarks:
base_states = base[bench]
variant_states = variant[bench]
state_names = set(base_states.keys())
if state_names != set(variant_states.keys()):
raise Exception("States do not match.")
result[bench] = {}
for state in state_names:
result[bench][state] = base_states[state] / variant_states[state]
return result
def values_to_space(axes):
result = []
for axis in axes:
result.append(["{}={}".format(axis, value) for value in axes[axis]])
return list(itertools.product(*result))
class ProcessRunner:
_instance = None
def __new__(cls, *args, **kwargs):
if not isinstance(cls._instance, cls):
cls._instance = super(ProcessRunner, cls).__new__(cls, *args, **kwargs)
return cls._instance
def __init__(self):
self.process = None
signal.signal(signal.SIGINT, self.signal_handler)
signal.signal(signal.SIGTERM, self.signal_handler)
def new_process(self, cmd):
self.process = subprocess.Popen(
cmd,
start_new_session=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
return self.process
def signal_handler(self, signum, frame):
self.kill_process()
raise SystemExit("search was interrupted")
def kill_process(self):
if self.process is not None:
self.process.kill()
class Bench:
def __init__(self, algorithm_name, variant, ct_workload):
self.algname = algorithm_name
self.variant = variant
self.ct_workload = ct_workload
def label(self):
return self.algname + "." + self.variant.label()
def variant_name(self):
return self.variant.label()
def algorithm_name(self):
return self.algname
def is_base(self):
return self.variant.is_base()
def get_base(self):
return BaseBench(self.algorithm_name())
def exe_name(self):
if self.is_base():
return self.algorithm_name() + ".base"
return self.algorithm_name() + ".variant"
def bench_names(self):
return [bench["name"] for bench in json_benches(self.algname)["benchmarks"]]
def axes_names(self):
subbench_names = {}
for bench in json_benches(self.algname)["benchmarks"]:
names = []
for axis in bench["axes"]:
names.append(get_axis_name(axis))
subbench_names[bench["name"]] = names
return subbench_names
def axes_values(self, sub_space, ct):
subbench_space = {}
for bench in json_benches(self.algname)["benchmarks"]:
space = {}
for axis in bench["axes"]:
name = get_axis_name(axis)
if ct:
if "{ct}" not in name:
continue
else:
if "{ct}" in name:
continue
axis_space = []
if name in sub_space:
for value in sub_space[name]:
axis_space.append(value)
else:
for value in axis["values"]:
axis_space.append(value["input_string"])
space[name] = axis_space
subbench_space[bench["name"]] = space
return subbench_space
def ct_axes_value_descriptions(self):
subbench_descriptions = {}
for bench in json_benches(self.algname)["benchmarks"]:
descriptions = {}
for axis in bench["axes"]:
name = axis["name"]
if "{ct}" not in name:
continue
if axis["flags"]:
name = name + "[{}]".format(axis["flags"])
descriptions[name] = {}
for value in axis["values"]:
descriptions[name][value["input_string"]] = value["description"]
subbench_descriptions[bench["name"]] = descriptions
return first_val(subbench_descriptions)
def axis_values(self, axis_name):
result = json_benches(self.algname)
if len(result["benchmarks"]) != 1:
raise Exception("Executable should contain exactly one benchmark")
for axis in result["benchmarks"][0]["axes"]:
name = axis["name"]
if axis["flags"]:
name = name + "[{}]".format(axis["flags"])
if name != axis_name:
continue
values = []
for value in axis["values"]:
values.append(value["input_string"])
return values
return []
def build(self):
if not self.is_base():
self.get_base().build()
build = CMake().build(self)
return build.code == 0
def definitions(self):
definitions = self.variant.tuning()
definitions = definitions + "\n"
descriptions = self.ct_axes_value_descriptions()
for ct_component in self.ct_workload:
ct_axis_name, ct_value = ct_component.split("=")
description = descriptions[ct_axis_name][ct_value]
ct_axis_name = ct_axis_name.replace("{ct}", "")
definitions = definitions + "#define TUNE_{} {}\n".format(
ct_axis_name, description
)
return definitions
def do_run(self, ct_point, rt_values, timeout, is_search=True):
logger = Logger()
try:
result_path = "result.json"
if os.path.exists(result_path):
os.remove(result_path)
bench_path = os.path.join(".", "bin", self.exe_name())
cmd = [bench_path]
for value in ct_point:
cmd.append("-a")
cmd.append(value)
cmd.append("--jsonbin")
cmd.append(result_path)
cmd.append("--stopping-criterion")
cmd.append("entropy")
# NVBench is currently broken for multiple GPUs, use `CUDA_VISIBLE_DEVICES`
cmd.append("-d")
cmd.append("0")
for bench in rt_values:
cmd.append("-b")
cmd.append(bench)
for axis in rt_values[bench]:
cmd.append("-a")
cmd.append("{}=[{}]".format(axis, ",".join(rt_values[bench][axis])))
logger.info(
"starting benchmark {} with {}: {}".format(
self.label(), ct_point, " ".join(cmd)
)
)
begin = time.time()
p = ProcessRunner().new_process(cmd)
p.wait(timeout=timeout)
elapsed = time.time() - begin
logger.info(
"finished benchmark {} with {} ({}) in {:.3f}s".format(
self.label(), ct_point, p.returncode, elapsed
)
)
return BenchResult(result_path, p.returncode, elapsed)
except subprocess.TimeoutExpired:
logger.info(
"benchmark {} with {} reached timeout of {:.3f}s".format(
self.label(), ct_point, timeout
)
)
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
return BenchResult(None, 42, float("inf"))
def ct_workload_space(self, sub_space):
if not self.build():
raise Exception("Unable to build benchmark: " + self.label())
return values_to_space(first_val(self.axes_values(sub_space, True)))
def rt_axes_values(self, sub_space):
if not self.build():
raise Exception("Unable to build benchmark: " + self.label())
return self.axes_values(sub_space, False)
def run(self, ct_workload_point, rt_values, estimator, is_search=True):
logger = Logger()
bench_cache = BenchCache()
runs_cache = RunsCache()
cached_centers = bench_cache.pull_bench_centers(
self, ct_workload_point, rt_values
)
if cached_centers:
logger.info("found benchmark {} in cache".format(self.label()))
return cached_centers
timeout = None
if not self.is_base():
code, elapsed = runs_cache.pull_run(self.get_base())
if code != 0:
raise Exception("Base bench return code = " + code)
timeout = elapsed * 50
result = self.do_run(ct_workload_point, rt_values, timeout, is_search)
runs_cache.push_run(self, result.code, result.elapsed)
return bench_cache.push_bench_centers(self, result, estimator)
def speedup(self, ct_workload_point, rt_values, base_estimator, variant_estimator):
if self.is_base():
return 1.0
base = self.get_base()
base_center = base.run(ct_workload_point, rt_values, base_estimator)
self_center = self.run(ct_workload_point, rt_values, variant_estimator)
return speedup(base_center, self_center)
def score(self, ct_workload, rt_values, base_estimator, variant_estimator):
if self.is_base():
return 1.0
speedups = self.speedup(
ct_workload, rt_values, base_estimator, variant_estimator
)
if not speedups:
return float("-inf")
rt_axes_ids = compute_axes_ids(rt_values)
weight_matrices = compute_weight_matrices(rt_values, rt_axes_ids)
score = 0
for bench in speedups:
for state in speedups[bench]:
rt_workload = state_to_rt_workload(bench, state)
weights = weight_matrices[bench]
weight = get_workload_weight(
rt_workload, rt_values[bench], rt_axes_ids[bench], weights
)
speedup = speedups[bench][state]
score = score + weight * speedup
return score
class BaseBench(Bench):
def __init__(self, algname):
super().__init__(algname, BasePoint(), [])

7
cccl_upstream/benchmarks/scripts/cccl/bench/build.py Normal file
View File

@@ -0,0 +1,7 @@
class Build:
def __init__(self, code, elapsed):
self.code = code
self.elapsed = elapsed
def __repr__(self):
return "Build(code = {}, elapsed = {:.4f}s)".format(self.code, self.elapsed)

138
cccl_upstream/benchmarks/scripts/cccl/bench/cmake.py Normal file
View File

@@ -0,0 +1,138 @@
import os
import signal
import subprocess
import time
from .build import Build
from .config import Config
from .logger import Logger
from .storage import Storage
def create_builds_table(conn):
with conn:
conn.execute("""
CREATE TABLE IF NOT EXISTS builds (
ctk TEXT NOT NULL,
cccl TEXT NOT NULL,
bench TEXT NOT NULL,
code TEXT NOT NULL,
elapsed REAL
);
""")
class CMakeCache:
_instance = None
def __new__(cls, *args, **kwargs):
if cls._instance is None:
cls._instance = super().__new__(cls, *args, **kwargs)
create_builds_table(Storage().connection())
return cls._instance
def pull_build(self, bench):
config = Config()
ctk = config.ctk
cccl = config.cccl
conn = Storage().connection()
with conn:
query = "SELECT code, elapsed FROM builds WHERE ctk = ? AND cccl = ? AND bench = ?;"
result = conn.execute(query, (ctk, cccl, bench.label())).fetchone()
if result:
code, elapsed = result
return Build(int(code), float(elapsed))
return result
def push_build(self, bench, build):
config = Config()
ctk = config.ctk
cccl = config.cccl
conn = Storage().connection()
with conn:
conn.execute(
"INSERT INTO builds (ctk, cccl, bench, code, elapsed) VALUES (?, ?, ?, ?, ?);",
(ctk, cccl, bench.label(), build.code, build.elapsed),
)
class CMake:
def __init__(self):
pass
def do_build(self, bench, timeout):
logger = Logger()
try:
if not bench.is_base():
with open(bench.exe_name() + ".h", "w") as f:
f.writelines(bench.definitions())
cmd = ["cmake", "--build", ".", "--target", bench.exe_name()]
logger.info(
"starting build for {}: {}".format(bench.label(), " ".join(cmd))
)
begin = time.time()
p = subprocess.Popen(
cmd,
start_new_session=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
p.wait(timeout=timeout)
elapsed = time.time() - begin
logger.info(
"finished build for {} (exit code: {}) in {:.3f}s".format(
bench.label(), p.returncode, elapsed
)
)
return Build(p.returncode, elapsed)
except subprocess.TimeoutExpired:
logger.info(
"build for {} reached timeout of {}s".format(bench.label(), timeout)
)
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
return Build(424242, float("inf"))
def build(self, bench):
logger = Logger()
timeout = None
cache = CMakeCache()
if bench.is_base():
# Only base build can be pulled from cache
build = cache.pull_build(bench)
if build:
logger.info("found cached base build for {}".format(bench.label()))
if bench.is_base():
if not os.path.exists("bin/{}".format(bench.exe_name())):
self.do_build(bench, None)
return build
else:
base_build = self.build(bench.get_base())
if base_build.code != 0:
raise Exception("Base build failed")
timeout = base_build.elapsed * 10
build = self.do_build(bench, timeout)
cache.push_build(bench, build)
return build
def clean():
cmd = ["cmake", "--build", ".", "--target", "clean"]
p = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
p.wait()
if p.returncode != 0:
raise Exception("Unable to clean build directory")

153
cccl_upstream/benchmarks/scripts/cccl/bench/config.py Normal file
View File

@@ -0,0 +1,153 @@
import os
import random
import sys
def randomized_cartesian_product(list_of_lists):
length = 1
for lst in list_of_lists:
length *= len(lst)
visited = set()
while len(visited) < length:
variant = tuple(map(random.choice, list_of_lists))
if variant not in visited:
visited.add(variant)
yield variant
class Range:
def __init__(self, definition, label, low, high, step):
self.definition = definition
self.label = label
self.low = low
self.high = high
self.step = step
class RangePoint:
def __init__(self, definition, label, value):
self.definition = definition
self.label = label
self.value = value
class VariantPoint:
def __init__(self, range_points):
self.range_points = range_points
def label(self):
if self.is_base():
return "base"
return ".".join(
["{}_{}".format(point.label, point.value) for point in self.range_points]
)
def is_base(self):
return len(self.range_points) == 0
def tuning(self):
if self.is_base():
return ""
tuning = "#pragma once\n\n"
for point in self.range_points:
tuning += "#define {} {}\n".format(point.definition, point.value)
return tuning
class BasePoint(VariantPoint):
def __init__(self):
VariantPoint.__init__(self, [])
def parse_ranges(columns):
ranges = []
for column in columns:
definition, label_range = column.split("|")
label, range = label_range.split("=")
start, end, step = [int(x) for x in range.split(":")]
ranges.append(Range(definition, label, start, end + 1, step))
return ranges
def parse_meta():
if not os.path.isfile("cccl_meta_bench.csv"):
print("cccl_meta_bench.csv not found", file=sys.stderr)
print(
"make sure to run the script from the CUB build directory", file=sys.stderr
)
benchmarks = {}
ctk_version = "0.0.0"
cccl_revision = "0.0-0-0000"
with open("cccl_meta_bench.csv", "r") as f:
lines = f.readlines()
for line in lines:
if "," in line:
columns = line.split(",")
else:
columns = [" ".join(line.split())]
name = columns[0]
if name == "ctk_version":
ctk_version = columns[1].rstrip()
elif name == "cccl_revision":
cccl_revision = columns[1].rstrip()
else:
if len(columns) > 1:
benchmarks[name] = parse_ranges(columns[1:])
else:
benchmarks[name] = []
return ctk_version, cccl_revision, benchmarks
class Config:
_instance = None
def __new__(cls, *args, **kwargs):
if cls._instance is None:
cls._instance = super().__new__(cls, *args, **kwargs)
cls._instance.ctk, cls._instance.cccl, cls._instance.benchmarks = (
parse_meta()
)
return cls._instance
def label_to_variant_point(self, algname, label):
if label == "base":
return BasePoint()
label_to_definition = {}
for param_space in self.benchmarks[algname]:
label_to_definition[param_space.label] = param_space.definition
points = []
for point in label.split("."):
label, value = point.split("_")
points.append(RangePoint(label_to_definition[label], label, int(value)))
return VariantPoint(points)
def variant_space(self, algname):
variants = []
for param_space in self.benchmarks[algname]:
variants.append([])
for value in range(param_space.low, param_space.high, param_space.step):
variants[-1].append(
RangePoint(param_space.definition, param_space.label, value)
)
return (
VariantPoint(points) for points in randomized_cartesian_product(variants)
)
def variant_space_size(self, algname):
num_variants = 1
for param_space in self.benchmarks[algname]:
num_variants = num_variants * len(
range(param_space.low, param_space.high, param_space.step)
)
return num_variants

20
cccl_upstream/benchmarks/scripts/cccl/bench/logger.py Normal file
View File

@@ -0,0 +1,20 @@
import logging
class Logger:
_instance = None
def __new__(cls, *args, **kwargs):
if cls._instance is None:
cls._instance = super().__new__(cls, *args, **kwargs)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
file_handler = logging.FileHandler("cccl_meta_bench.log")
file_handler.setFormatter(logging.Formatter("%(asctime)s: %(message)s"))
logger.addHandler(file_handler)
cls._instance.logger = logger
return cls._instance
def info(self, message):
self.logger.info(message)

105
cccl_upstream/benchmarks/scripts/cccl/bench/score.py Normal file
View File

@@ -0,0 +1,105 @@
import math
import numpy as np
def importance_function(x):
return 1 - math.exp(-x)
def x_by_importance(y):
return -math.log(1 - y)
def compute_weights(num_values):
least_importance = 0.6
most_importance = 0.999
assert least_importance < most_importance
assert least_importance >= 0 and least_importance < 1
assert most_importance > 0 and most_importance < 1
begin = x_by_importance(least_importance)
end = x_by_importance(most_importance)
rng = end - begin
step = rng / num_values
return np.array([importance_function(begin + x * step) for x in range(num_values)])
def io_weights(values):
return compute_weights(len(values))
def ei_weights(values):
return np.ones(len(values))
def compute_axes_ids(rt_axes_values):
result = {}
for bench in rt_axes_values:
rt_axes_ids = {}
axis_id = 0
for rt_axis in rt_axes_values[bench]:
rt_axes_ids[rt_axis] = axis_id
axis_id = axis_id + 1
result[bench] = rt_axes_ids
return result
def compute_raw_weight_matrix(rt_axes_values, rt_axes_ids):
rt_axes_weights = {}
first_rt_axis = True
first_rt_axis_name = None
for rt_axis, values in rt_axes_values.items():
if first_rt_axis:
first_rt_axis_name = rt_axis
first_rt_axis = False
if "{io}" in rt_axis:
rt_axes_weights[rt_axis] = io_weights(values)
else:
rt_axes_weights[rt_axis] = ei_weights(values)
num_rt_axes = len(rt_axes_ids)
for rt_axis in rt_axes_weights:
shape = [1] * num_rt_axes
shape[rt_axes_ids[rt_axis]] = -1
rt_axes_weights[rt_axis] = rt_axes_weights[rt_axis].reshape(*shape)
weights_matrix = rt_axes_weights[first_rt_axis_name]
for rt_axis in rt_axes_weights:
if rt_axis == first_rt_axis_name:
continue
weights_matrix = weights_matrix * rt_axes_weights[rt_axis]
return weights_matrix
def compute_weight_matrices(rt_axes_values, rt_axes_ids):
matrices = {}
aggregate = 0.0
for bench in rt_axes_values:
matrices[bench] = compute_raw_weight_matrix(
rt_axes_values[bench], rt_axes_ids[bench]
)
aggregate = aggregate + np.sum(matrices[bench])
for bench in rt_axes_values:
matrices[bench] = matrices[bench] / aggregate
return matrices
def get_workload_coordinates(rt_workload, rt_axes_values, rt_axes_ids):
coordinates = [0] * len(rt_axes_ids)
for point in rt_workload:
rt_axis, rt_value = point.split("=")
coordinates[rt_axes_ids[rt_axis]] = rt_axes_values[rt_axis].index(rt_value)
return coordinates
def get_workload_weight(rt_workload, rt_axes_values, rt_axes_ids, weights_matrix):
coordinates = get_workload_coordinates(rt_workload, rt_axes_values, rt_axes_ids)
return weights_matrix[tuple(coordinates)]

214
cccl_upstream/benchmarks/scripts/cccl/bench/search.py Normal file
View File

@@ -0,0 +1,214 @@
import argparse
import re
import numpy as np
from .bench import BaseBench, Bench
from .cmake import CMake
from .config import Config
from .storage import Storage
def list_benches(algnames):
print("### Benchmarks")
config = Config()
for algname in algnames:
space_size = config.variant_space_size(algname)
print(" * `{}`: {} variants: ".format(algname, space_size))
for param_space in config.benchmarks[algname]:
param_name = param_space.label
param_rng = (param_space.low, param_space.high, param_space.step)
print(" * `{}`: {}".format(param_name, param_rng))
def parse_sub_space(args):
sub_space = {}
for axis in args:
name, value = axis.split("=")
if "[" in value:
value = value.replace("[", "").replace("]", "")
values = value.split(",")
else:
values = [value]
sub_space[name] = values
return sub_space
def parse_arguments():
parser = argparse.ArgumentParser(
description="Runs benchmarks and stores results in a database."
)
parser.add_argument(
"-R", type=str, default=".*", help="Regex for benchmarks selection."
)
parser.add_argument(
"-a",
"--args",
action="append",
type=str,
help="Parameter in the format `Param=Value`.",
)
parser.add_argument(
"-l", "--list-benches", action="store_true", help="Show available benchmarks."
)
parser.add_argument(
"--num-shards",
type=int,
default=1,
help="Split benchmarks into NUM_SHARDS pieces and only run one",
)
parser.add_argument(
"--run-shard",
type=int,
default=0,
help="Run benchmark shard RUN_SHARD from NUM_SHARDS pieces",
)
parser.add_argument("-P0", action="store_true", help="Run P0 benchmarks")
return parser.parse_args()
def filter_benchmark_space_for_p0(algname, ct_space, rt_values):
if algname in [
"cub.bench.merge_sort.pairs",
"cub.bench.radix_sort.pairs",
"cub.bench.select.unique_by_key",
]:
ct_space = list(
filter(
lambda variant: not (
("OffsetT{ct}=I64" in variant)
or ("KeyT{ct}=I16" in variant)
or ("ValueT{ct}=I16" in variant)
or ("KeyT{ct}=I128" in variant)
or ("ValueT{ct}=I128" in variant)
),
ct_space,
)
)
if algname == "cub.bench.merge_sort.pairs":
for subbench in rt_values:
for axis in rt_values[subbench]:
if axis == "Entropy":
rt_values[subbench][axis] = ["1.000"]
return ct_space, rt_values
def run_benches(algnames, sub_space, seeker, args):
for algname in algnames:
try:
bench = BaseBench(algname)
ct_space = bench.ct_workload_space(sub_space)
rt_values = bench.rt_axes_values(sub_space)
if args.P0:
ct_space, rt_values = filter_benchmark_space_for_p0(
algname, ct_space, rt_values
)
seeker(algname, ct_space, rt_values)
except Exception as e:
print(
"#### ERROR exception occurred while running {}: '{}'".format(
algname, e
)
)
def filter_benchmarks_by_regex(benchmarks, R):
pattern = re.compile(R)
return list(filter(lambda x: pattern.match(x), benchmarks))
def filter_benchmarks(benchmarks, args):
if args.run_shard >= args.num_shards:
raise ValueError("run-shard must be less than num-shards")
p0_benchmarks = [
"cub.bench.merge_sort.keys",
"cub.bench.merge_sort.pairs",
"cub.bench.radix_sort.keys",
"cub.bench.radix_sort.pairs",
"cub.bench.reduce.by_key",
"cub.bench.reduce.custom",
"cub.bench.reduce.sum",
"cub.bench.scan.exclusive.deterministic",
"cub.bench.scan.exclusive.sum",
"cub.bench.select.flagged",
"cub.bench.select.if",
"cub.bench.select.unique",
"cub.bench.select.unique_by_key",
"cub.bench.transform.babelstream",
"cub.bench.transform.fill",
]
algnames = filter_benchmarks_by_regex(benchmarks.keys(), args.R)
if args.P0:
algnames = [name for name in p0_benchmarks if name in algnames]
algnames.sort()
if args.num_shards > 1:
algnames = np.array_split(algnames, args.num_shards)[args.run_shard].tolist()
return algnames
return algnames
def search(seeker):
args = parse_arguments()
if not Storage().exists():
CMake().clean()
config = Config()
print(" ctk: ", config.ctk)
print("cccl: ", config.cccl)
workload_sub_space = {}
if args.args:
workload_sub_space = parse_sub_space(args.args)
algnames = filter_benchmarks(config.benchmarks, args)
if args.list_benches:
list_benches(algnames)
return
run_benches(algnames, workload_sub_space, seeker, args)
class MedianCenterEstimator:
def __init__(self):
pass
def __call__(self, samples):
if len(samples) == 0:
return float("inf")
return float(np.median(samples))
class BruteForceSeeker:
def __init__(self, base_center_estimator, variant_center_estimator):
self.base_center_estimator = base_center_estimator
self.variant_center_estimator = variant_center_estimator
def __call__(self, algname, ct_workload_space, rt_values):
variants = Config().variant_space(algname)
for ct_workload in ct_workload_space:
for variant in variants:
bench = Bench(algname, variant, list(ct_workload))
if bench.build():
score = bench.score(
ct_workload,
rt_values,
self.base_center_estimator,
self.variant_center_estimator,
)
print(bench.label(), score)

392
cccl_upstream/benchmarks/scripts/cccl/bench/storage.py Normal file
View File

@@ -0,0 +1,392 @@
import os
import sqlite3
import fpzip
import numpy as np
import pandas as pd
db_name = "cccl_meta_bench.db"
# PostgreSQL support
try:
import psycopg2
import psycopg2.extras
POSTGRES_AVAILABLE = True
except ImportError:
POSTGRES_AVAILABLE = False
def get_postgres_config():
"""Get PostgreSQL configuration from environment variables."""
if not POSTGRES_AVAILABLE:
return None
# Check if all required environment variables are set
required_vars = [
"CCCL_BENCH_PG_HOST",
"CCCL_BENCH_PG_USER",
"CCCL_BENCH_PG_DB",
"CCCL_BENCH_PG_PASSWORD",
]
config = {}
for var in required_vars:
value = os.environ.get(var)
if not value:
return None # Fall back to SQLite if any required var is missing
config[var] = value
# Optional port (default to 5432)
config["CCCL_BENCH_PG_PORT"] = os.environ.get("CCCL_BENCH_PG_PORT", "5432")
return config
def get_bench_table_name(subbench, algname):
return "{}.{}".format(algname, subbench)
def blob_to_samples(blob):
return np.squeeze(fpzip.decompress(blob))
class StorageBase:
"""Abstract base class for storage backends."""
def connection(self):
raise NotImplementedError
def exists(self):
raise NotImplementedError
def algnames(self):
raise NotImplementedError
def subbenches(self, algname):
raise NotImplementedError
def alg_to_df(self, algname, subbench):
raise NotImplementedError
def store_df(self, algname, df):
raise NotImplementedError
class SQLiteStorage(StorageBase):
def __init__(self, db_path):
self.db_path = db_path
self.conn = sqlite3.connect(db_path)
def connection(self):
return self.conn
def exists(self):
return os.path.exists(self.db_path)
def algnames(self):
with self.conn:
rows = self.conn.execute(
"SELECT DISTINCT algorithm FROM subbenches"
).fetchall()
return [row[0] for row in rows]
def subbenches(self, algname):
with self.conn:
rows = self.conn.execute(
"SELECT DISTINCT bench FROM subbenches WHERE algorithm=?", (algname,)
).fetchall()
return [row[0] for row in rows]
def alg_to_df(self, algname, subbench):
table = get_bench_table_name(subbench, algname)
with self.conn:
df = pd.read_sql_query('SELECT * FROM "{}"'.format(table), self.conn)
df["samples"] = df["samples"].apply(blob_to_samples)
return df
def store_df(self, algname, df):
df["samples"] = df["samples"].apply(fpzip.compress)
df.to_sql(algname, self.conn, if_exists="replace", index=False)
class PostgreSQLConnectionWrapper:
"""Wrapper to make psycopg2 connection compatible with sqlite3 interface."""
def __init__(self, pg_conn):
self.pg_conn = pg_conn
self.pg_conn.autocommit = False
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is None:
self.pg_conn.commit()
else:
self.pg_conn.rollback()
def execute(self, query, params=None):
"""Execute query with SQLite-style parameter substitution."""
# Convert SQLite-style ? placeholders to PostgreSQL %s
if params:
query = query.replace("?", "%s")
# Convert SQLite BLOB type to PostgreSQL BYTEA
query = query.replace(" BLOB", " BYTEA")
# Fix SQLite-style double-quoted string literals to PostgreSQL single quotes
# This is a simple approach - in production you'd want a proper SQL parser
import re
# Match patterns like = "value" and convert to = 'value'
query = re.sub(r'= "([^"]*)"', r"= '\1'", query)
# Handle ON CONFLICT DO NOTHING (SQLite) -> ON CONFLICT DO NOTHING (PostgreSQL)
# Both databases support this syntax, so no conversion needed
cur = self.pg_conn.cursor()
if params:
cur.execute(query, params)
else:
cur.execute(query)
return cur
def commit(self):
self.pg_conn.commit()
def rollback(self):
self.pg_conn.rollback()
def close(self):
self.pg_conn.close()
if POSTGRES_AVAILABLE:
class PostgreSQLStorage(StorageBase):
def __init__(self, config):
self.config = config
self.pg_conn = psycopg2.connect(
host=config["CCCL_BENCH_PG_HOST"],
port=config["CCCL_BENCH_PG_PORT"],
database=config["CCCL_BENCH_PG_DB"],
user=config["CCCL_BENCH_PG_USER"],
password=config["CCCL_BENCH_PG_PASSWORD"],
)
self.conn = PostgreSQLConnectionWrapper(self.pg_conn)
def connection(self):
return self.conn
def exists(self):
# For PostgreSQL, check if the subbenches table exists
with self.conn:
cur = self.conn.execute("""
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_name = 'subbenches'
);
""")
return cur.fetchone()[0]
def algnames(self):
with self.conn:
cur = self.conn.execute("SELECT DISTINCT algorithm FROM subbenches")
rows = cur.fetchall()
return [row[0] for row in rows]
def subbenches(self, algname):
with self.conn:
cur = self.conn.execute(
"SELECT DISTINCT bench FROM subbenches WHERE algorithm=?",
(algname,),
)
rows = cur.fetchall()
return [row[0] for row in rows]
def alg_to_df(self, algname, subbench):
table = get_bench_table_name(subbench, algname)
with self.conn:
# Use proper quoting for PostgreSQL
query = 'SELECT * FROM "{}"'.format(table.replace('"', '""'))
df = pd.read_sql_query(query, self.pg_conn)
df["samples"] = df["samples"].apply(lambda x: blob_to_samples(bytes(x)))
return df
def store_df(self, algname, df):
df["samples"] = df["samples"].apply(fpzip.compress)
# For PostgreSQL, we need to use a different approach
# as pandas doesn't support direct to_sql with psycopg2
# We'll need to implement this separately or use SQLAlchemy
raise NotImplementedError(
"DataFrame storage for PostgreSQL not yet implemented"
)
else:
# Define a dummy class when psycopg2 is not available
PostgreSQLStorage = None
class DualStorageWrapper:
"""Wrapper that writes to multiple storage backends."""
def __init__(self, primary, secondary=None):
self.primary = primary
self.secondary = secondary
self._primary_conn = None
self._secondary_conn = None
def connection(self):
# Return a wrapper that forwards operations to both backends
if not self._primary_conn:
self._primary_conn = DualConnectionWrapper(
self.primary.connection(),
self.secondary.connection() if self.secondary else None,
)
return self._primary_conn
def exists(self):
# Check primary storage
return self.primary.exists()
def algnames(self):
# Read from primary only
return self.primary.algnames()
def subbenches(self, algname):
# Read from primary only
return self.primary.subbenches(algname)
def alg_to_df(self, algname, subbench):
# Read from primary only
return self.primary.alg_to_df(algname, subbench)
def store_df(self, algname, df):
# Write to both databases
self.primary.store_df(algname, df)
if self.secondary:
try:
self.secondary.store_df(algname, df)
except Exception as e:
print(f"Warning: Failed to write to secondary storage: {e}")
class DualCursorWrapper:
"""Wrapper for cursor results from dual storage."""
def __init__(self, primary_cursor):
self.primary_cursor = primary_cursor
def fetchone(self):
return self.primary_cursor.fetchone()
def fetchall(self):
return self.primary_cursor.fetchall()
class DualConnectionWrapper:
"""Wrapper that forwards connection operations to both backends."""
def __init__(self, primary_conn, secondary_conn=None):
self.primary_conn = primary_conn
self.secondary_conn = secondary_conn
def __enter__(self):
# SQLite connections are their own context managers
return self
def __exit__(self, exc_type, exc_val, exc_tb):
# Commit or rollback based on exception
if exc_type is None:
self.commit()
else:
self.rollback()
return False
def execute(self, query, params=None):
# Execute on primary
if params:
primary_result = self.primary_conn.execute(query, params)
else:
primary_result = self.primary_conn.execute(query)
# Also execute on secondary if available
if self.secondary_conn:
try:
if params:
self.secondary_conn.execute(query, params)
else:
self.secondary_conn.execute(query)
except Exception:
# Don't print warnings for every query, too noisy
pass
# Return a wrapper that delegates to the primary result
return DualCursorWrapper(primary_result)
def fetchone(self):
# Delegate to primary connection
return self.primary_conn.fetchone()
def fetchall(self):
# Delegate to primary connection
return self.primary_conn.fetchall()
def commit(self):
self.primary_conn.commit()
if self.secondary_conn:
try:
self.secondary_conn.commit()
except Exception as e:
print(f"Warning: Failed to commit to secondary storage: {e}")
def rollback(self):
self.primary_conn.rollback()
if self.secondary_conn:
try:
self.secondary_conn.rollback()
except Exception as e:
print(f"Warning: Failed to rollback secondary storage: {e}")
class Storage:
_instance = None
def __new__(cls, *args, **kwargs):
if cls._instance is None:
cls._instance = super().__new__(cls, *args, **kwargs)
# Always use SQLite as primary
sqlite_storage = SQLiteStorage(db_name)
# Try to add PostgreSQL as secondary if configured
pg_config = get_postgres_config()
pg_storage = None
if pg_config and PostgreSQLStorage is not None:
try:
pg_storage = PostgreSQLStorage(pg_config)
print(
"Using dual storage: SQLite (primary) + PostgreSQL (secondary)"
)
except Exception as e:
print(f"Failed to connect to PostgreSQL: {e}")
print("Using SQLite only")
# Create wrapper with SQLite as primary and PostgreSQL as optional secondary
cls._instance.base = DualStorageWrapper(sqlite_storage, pg_storage)
return cls._instance
def connection(self):
return self.base.connection()
def exists(self):
return self.base.exists()
def algnames(self):
return self.base.algnames()
def alg_to_df(self, algname, subbench):
return self.base.alg_to_df(algname, subbench)

158
cccl_upstream/benchmarks/scripts/compare.py Executable file
View File

@@ -0,0 +1,158 @@
#!/usr/bin/env python3
import argparse
import os
import cccl
import numpy as np
import pandas as pd
from colorama import Fore
def get_filenames_map(arr):
if not arr:
return []
prefix = arr[0]
for string in arr:
while not string.startswith(prefix):
prefix = prefix[:-1]
if not prefix:
break
return {string: string[len(prefix) :] for string in arr}
def is_finite(x):
if isinstance(x, float):
return x != np.inf and x != -np.inf
return True
def filter_by_problem_size(df):
min_elements_pow2 = 28
if "Elements{io}[pow2]" in df.columns:
df["Elements{io}[pow2]"] = df["Elements{io}[pow2]"].astype(int)
df = df[df["Elements{io}[pow2]"] >= min_elements_pow2]
return df
def filter_by_offset_type(df):
if "OffsetT{ct}" in df.columns:
df = df[(df["OffsetT{ct}"] == "I32") | (df["OffsetT{ct}"] == "U32")]
return df
def filter_by_type(df):
if "T{ct}" in df:
# df = df[df['T{ct}'].str.contains('64')]
df = df[~df["T{ct}"].str.contains("C")]
elif "KeyT{ct}" in df:
# df = df[df['KeyT{ct}'].str.contains('64')]
df = df[~df["KeyT{ct}"].str.contains("C")]
return df
def alg_dfs(file):
result = {}
storage = cccl.bench.SQLiteStorage(file)
for algname in storage.algnames():
for subbench in storage.subbenches(algname):
df = storage.alg_to_df(algname, subbench)
df = df.map(lambda x: x if is_finite(x) else np.nan)
df = df.dropna(subset=["center"], how="all")
# TODO(bgruber): maybe expose the filters under a -p0, or --short flag
# df = filter_by_type(filter_by_offset_type(filter_by_problem_size(df)))
df["Noise"] = df["samples"].apply(lambda x: np.std(x) / np.mean(x)) * 100
df["Mean"] = df["samples"].apply(lambda x: np.mean(x))
df = df.drop(columns=["samples", "center", "bw", "elapsed", "variant"])
fused_algname = (
algname.removeprefix("cub.bench.").removeprefix("thrust.bench.")
+ "."
+ subbench
)
result[fused_algname] = df
for algname in result:
if result[algname]["cccl"].nunique() != 1:
print(f"WARNING: Multiple CCCL versions in one db '{algname}'")
result[algname] = result[algname].drop(columns=["cccl"])
return result
def file_exists(value):
if not os.path.isfile(value):
raise argparse.ArgumentTypeError(f"The file '{value}' does not exist.")
return value
def parse_args():
parser = argparse.ArgumentParser(description="Analyze benchmark results.")
parser.add_argument("reference", type=file_exists, help="Reference database file.")
parser.add_argument("compare", type=file_exists, help="Comparison database file.")
return parser.parse_args()
config_count = 0
pass_count = 0
faster_count = 0
slower_count = 0
def status(frac_diff, noise_ref, noise_cmp):
global config_count
global pass_count
global faster_count
global slower_count
config_count += 1
min_noise = min(noise_ref, noise_cmp)
if abs(frac_diff) <= min_noise:
pass_count += 1
return Fore.BLUE + "SAME" + Fore.RESET
if frac_diff < 0:
faster_count += 1
return Fore.GREEN + "FAST" + Fore.RESET
if frac_diff > 0:
slower_count += 1
return Fore.RED + "SLOW" + Fore.RESET
def compare():
args = parse_args()
reference_df = alg_dfs(args.reference)
compare_df = alg_dfs(args.compare)
for alg in sorted(reference_df.keys() & compare_df.keys()):
print()
print()
print(f"# {alg}")
# use every column except 'Noise', 'Mean', 'ctk', 'gpu' to match runs between reference and comparison file
merge_columns = [
col
for col in reference_df[alg].columns
if col not in ["Noise", "Mean", "ctk", "gpu"]
]
df = pd.merge(
reference_df[alg],
compare_df[alg],
on=merge_columns,
suffixes=("Ref", "Cmp"),
)
df["Abs. Diff"] = df["MeanCmp"] - df["MeanRef"]
df["Rel. Diff"] = (df["Abs. Diff"] / df["MeanRef"]) * 100
df["Status"] = list(
map(status, df["Rel. Diff"], df["NoiseRef"], df["NoiseCmp"])
)
df = df.drop(columns=["ctkRef", "ctkCmp", "gpuRef", "gpuCmp"])
print()
print(df.to_markdown(index=False))
print("# Summary\n")
print("- Total Matches: %d" % config_count)
print(" - Pass (diff <= min_noise): %d" % pass_count)
print(" - Faster (diff > min_noise): %d" % faster_count)
print(" - Slower (diff > min_noise): %d" % slower_count)
if __name__ == "__main__":
compare()

81
cccl_upstream/benchmarks/scripts/run.py Executable file
View File

@@ -0,0 +1,81 @@
#!/usr/bin/env python3
import math
import os
import sys
import cccl.bench
def elapsed_time_looks_good(x):
if isinstance(x, float):
if math.isfinite(x):
return True
return False
def get_largest_problem_size(rt_values):
# Small problem sizes do not utilize entire GPU.
# Benchmarking small problem sizes in environments where we do not control
# distributions comparison, e.g. CI, is not useful because of stability issues.
elements = []
for element in rt_values:
if element.isdigit():
elements.append(int(element))
return [str(max(elements))]
def filter_runtime_workloads_for_ci(rt_values):
for subbench in rt_values:
for axis in rt_values[subbench]:
if axis.startswith("Elements") and axis.endswith("[pow2]"):
rt_values[subbench][axis] = get_largest_problem_size(
rt_values[subbench][axis]
)
return rt_values
class BaseRunner:
def __init__(self):
self.estimator = cccl.bench.MedianCenterEstimator()
def __call__(self, algname, ct_workload_space, rt_values):
failure_occured = False
rt_values = filter_runtime_workloads_for_ci(rt_values)
for ct_workload in ct_workload_space:
bench = cccl.bench.BaseBench(algname)
if bench.build(): # might throw
results = bench.run(ct_workload, rt_values, self.estimator, False)
for subbench in results:
for point in results[subbench]:
bench_name = "{}.{}-{}".format(
bench.algorithm_name(), subbench, point
)
bench_name = bench_name.replace(" ", "___")
bench_name = "".join(
c if c.isalnum() else "_" for c in bench_name
)
elapsed_time = results[subbench][point]
if elapsed_time_looks_good(elapsed_time):
print(
"&&&& PERF {} {} -sec".format(bench_name, elapsed_time)
)
else:
failure_occured = True
print("&&&& FAILED {}".format(algname))
if failure_occured:
sys.exit(1)
def main():
print("&&&& RUNNING bench")
os.environ["CUDA_MODULE_LOADING"] = "EAGER"
cccl.bench.search(BaseRunner())
print("&&&& PASSED bench")
if __name__ == "__main__":
main()

18
cccl_upstream/benchmarks/scripts/search.py Executable file
View File

@@ -0,0 +1,18 @@
#!/usr/bin/env python3
import cccl.bench as bench
# TODO:
# - driver version
# - host compiler + version
# - gpu clocks / pm
# - ecc
def main():
center_estimator = bench.MedianCenterEstimator()
bench.search(bench.BruteForceSeeker(center_estimator, center_estimator))
if __name__ == "__main__":
main()

184
cccl_upstream/benchmarks/scripts/sol.py Executable file
View File

@@ -0,0 +1,184 @@
#!/usr/bin/env python3
import argparse
import os
import re
import cccl
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
def is_finite(x):
if isinstance(x, float):
return x != np.inf and x != -np.inf
return True
def filter_by_problem_size(df):
min_elements_pow2 = 28
if "Elements{io}[pow2]" in df.columns:
df["Elements{io}[pow2]"] = df["Elements{io}[pow2]"].astype(int)
df = df[df["Elements{io}[pow2]"] >= min_elements_pow2]
return df
def filter_by_offset_type(df):
if "OffsetT{ct}" in df.columns:
filtered = df[
(df["OffsetT{ct}"] == "I32") | (df["OffsetT{ct}"] == "U32")
] # only use 32-bit offset types
if not filtered.empty: # some benchmarks only use a 64-bit offset type
df = filtered
return df
def filter_by_type(df):
if "T{ct}" in df:
# df = df[df['T{ct}'].str.contains('64')]
df = df[~df["T{ct}"].str.contains("C")]
elif "KeyT{ct}" in df:
# df = df[df['KeyT{ct}'].str.contains('64')]
df = df[~df["KeyT{ct}"].str.contains("C")]
return df
def alg_dfs(files, alg_regex):
pattern = re.compile(alg_regex)
result = {}
for file in files:
storage = cccl.bench.SQLiteStorage(file)
for algname in storage.algnames():
if pattern.match(algname):
for subbench in storage.subbenches(algname):
df = storage.alg_to_df(algname, subbench)
df = df.map(lambda x: x if is_finite(x) else np.nan)
df = df.dropna(subset=["center"], how="all")
df = filter_by_type(
filter_by_offset_type(filter_by_problem_size(df))
)
df = df.filter(items=["ctk", "cccl", "gpu", "variant", "bw"])
fused_algname = algname.replace("bench.", "") + "." + subbench
if df.empty:
print(
f"WARNING: Skipped {fused_algname} because no data is present"
)
print(df)
continue
if df["bw"].dropna().empty:
print(
f"WARNING: Skipped {fused_algname} because it does not report bandwidth"
)
continue
df["variant"] = df["variant"].astype(str)
df["bw"] = df["bw"] * 100
if fused_algname in result:
result[fused_algname] = pd.concat([result[fused_algname], df])
else:
result[fused_algname] = df
print(fused_algname)
return result
def alg_bws(dfs, verbose):
medians = None
for algname in dfs:
df = dfs[algname]
df["alg"] = algname
if df is None:
medians = df
else:
medians = pd.concat([medians, df])
# print more information if it's not unique across all runs or when requested (verbose)
medians["hue"] = ""
if verbose or medians["cccl"].unique().size > 1:
medians["hue"] = medians["hue"] + "CCCL " + medians["cccl"].astype(str) + " "
gpuname = (
medians["gpu"]
if verbose
else medians["gpu"].astype(str).map(lambda x: x[: x.find("(") - 1])
)
medians["hue"] = medians["hue"] + gpuname + " "
if medians["variant"].unique().size > 1:
variant = (
medians["variant"]
.astype(str)
.map(lambda x: (" " + x if x != "base" else ""))
)
medians["hue"] = medians["hue"] + variant + " "
if verbose or medians["ctk"].unique().size > 1:
medians["hue"] = medians["hue"] + "CTK " + medians["ctk"].astype(str)
return medians.drop(columns=["ctk", "cccl", "gpu", "variant"])
def file_exists(value):
if not os.path.isfile(value):
raise argparse.ArgumentTypeError(f"The file '{value}' does not exist.")
return value
def plot_sol(medians, box):
if box:
ax = sns.boxenplot(data=medians, x="alg", y="bw", hue="hue")
else:
ax = sns.barplot(
data=medians,
x="alg",
y="bw",
hue="hue",
errorbar=lambda x: (x.min(), x.max()),
)
ax.bar_label(ax.containers[0], fmt="%.1f")
for container in ax.containers[1:]:
labels = [
f"{c:.1f}\n({(c / f) * 100:.0f}%)"
for f, c in zip(ax.containers[0].datavalues, container.datavalues)
]
ax.bar_label(container, labels=labels)
ax.legend(title=None)
ax.set_xlabel("Algorithm")
ax.set_ylabel("Bandwidth (%SOL)")
ax.set_xticklabels(
ax.get_xticklabels(), rotation=30, rotation_mode="anchor", ha="right"
)
ax.set_ylim([0, 100])
plt.show()
def print_speedup(medians):
m = medians.groupby(["alg", "hue"], sort=False).mean()
m["speedup"] = m["bw"] / m.groupby(["alg"])["bw"].transform("first")
print("# Speedups:")
print()
print(m.drop(columns="bw").sort_values(by="speedup", ascending=False).to_markdown())
def parse_args():
parser = argparse.ArgumentParser(description="Analyze benchmark results.")
parser.add_argument(
"files", type=file_exists, nargs="+", help="At least one file is required."
)
parser.add_argument("--box", action="store_true", help="Plot box instead of bar.")
parser.add_argument("-v", action="store_true", help="Verbose legend.")
parser.add_argument(
"-R", type=str, default=".*", help="Regex for benchmarks selection."
)
return parser.parse_args()
def sol():
args = parse_args()
dfs = alg_dfs(args.files, args.R)
if not dfs:
print("ERROR: No benchmark data to process (all benchmarks were skipped).")
return
medians = alg_bws(dfs, args.v)
print_speedup(medians)
plot_sol(medians, args.box)
if __name__ == "__main__":
sol()

47
cccl_upstream/benchmarks/scripts/submit_benchmark_job.sh Executable file
View File

@@ -0,0 +1,47 @@
#!/usr/bin/env bash
# This script schedules a SLURM job via crun on computelab to run all CCCL benchmarks and produce a benchmark database
# TODO: set those accordingly
scratch=/home/scratch."$USER"_sw
node_selector="cpu.arch=x86_64 and gpu.product_name='*B200*'"
container_image="rapidsai/devcontainers:26.06-cpp-gcc14-cuda13.2"
jobtime="4:00:00"
benchmark_preset="benchmark"
batch_script=$scratch/batch.sh
cat << BATCH_SCRIPT > "$batch_script"
#!/usr/bin/env bash
pip install --break-system-packages fpzip pandas scipy
# clone CCCL
host=\$(hostname)
cd $scratch
if [ -d "\$host/cccl" ]; then
rm -r \$host/cccl
fi
mkdir \$host
cd \$host
git clone --depth 1 git@github.com:NVIDIA/cccl.git
cd cccl
# configure cmake
mkdir build_perf
cd build_perf
cmake .. --preset $benchmark_preset
# run benchmarks
export CUDA_VISIBLE_DEVICES=0
export PYTHONPATH=../benchmarks/scripts/
../benchmarks/scripts/run.py
echo "Benchmark done. Results in $scratch/\$host/cccl/build_perf/cccl_meta_bench.db"
BATCH_SCRIPT
chmod +x "$batch_script"
# schedule SLURM job
echo "Scheduling script $batch_script"
echo "#################################################################################"
cat "$batch_script"
echo "#################################################################################"
crun -q "$node_selector" -ex -t "$jobtime" -img "$container_image" -b "$batch_script"

72
cccl_upstream/benchmarks/scripts/verify.py Executable file
View File

@@ -0,0 +1,72 @@
#!/usr/bin/env python3
import argparse
import sys
import cccl.bench
def parse_arguments():
parser = argparse.ArgumentParser(description="Verify tuning variant")
parser.add_argument(
"--variant", type=str, help="Variant to verify", default=None, required=True
)
variant = parser.parse_known_args()[0].variant
sys.argv.remove("--variant={}".format(variant))
return variant
def workload_header(ct_workload_space, rt_workload_space):
for ct_workload in ct_workload_space:
for rt_workload in rt_workload_space:
workload_point = ct_workload + rt_workload
return ", ".join([x.split("=")[0] for x in workload_point])
def workload_entry(ct_workload, rt_workload):
workload_point = ct_workload + rt_workload
return ", ".join([x.split("=")[1] for x in workload_point])
class VerifySeeker:
def __init__(self, variant_label):
self.label = variant_label
self.estimator = cccl.bench.MedianCenterEstimator()
def __call__(self, algname, ct_workload_space, rt_workload_space):
variant_point = cccl.bench.Config().label_to_variant_point(algname, self.label)
print(
"{}, MinS, MedianS, MaxS".format(
workload_header(ct_workload_space, rt_workload_space)
)
)
for ct_workload in ct_workload_space:
bench = cccl.bench.Bench(algname, variant_point, list(ct_workload))
if bench.build():
base = bench.get_base()
for rt_workload in rt_workload_space:
workload_point = ct_workload + rt_workload
base_samples, base_elapsed = base.do_run(workload_point, None)
variant_samples, _ = bench.do_run(workload_point, base_elapsed * 10)
min_speedup = min(base_samples) / min(variant_samples)
median_speedup = self.estimator(base_samples) / self.estimator(
variant_samples
)
max_speedup = max(base_samples) / max(variant_samples)
point_str = workload_entry(ct_workload, rt_workload)
print(
"{}, {}, {}, {}".format(
point_str, min_speedup, median_speedup, max_speedup
)
)
def main():
cccl.bench.search(VerifySeeker(parse_arguments()))
if __name__ == "__main__":
main()

19
cccl_upstream/c/CMakeLists.txt Normal file
View File

@@ -0,0 +1,19 @@
if (CCCL_ENABLE_C_PARALLEL AND CCCL_ENABLE_C_PARALLEL_V2)
message(
FATAL_ERROR
"CCCL_ENABLE_C_PARALLEL and CCCL_ENABLE_C_PARALLEL_V2 are mutually exclusive. "
"v2 is the HostJIT-based successor of v1; pick one."
)
endif()
if (CCCL_ENABLE_C_PARALLEL)
add_subdirectory(parallel)
endif()
if (CCCL_ENABLE_C_PARALLEL_V2)
add_subdirectory(parallel.v2)
endif()
if (CCCL_ENABLE_C_EXPERIMENTAL_STF)
add_subdirectory(experimental/stf)
endif()

79
cccl_upstream/c/experimental/stf/CMakeLists.txt Normal file
View File

@@ -0,0 +1,79 @@
cmake_minimum_required(VERSION 3.21)
project(CCCL_C_EXPERIMENTAL_STF LANGUAGES CUDA CXX C)
option(
CCCL_C_EXPERIMENTAL_STF_ENABLE_TESTING
"Build cccl.experimental.c.stf tests."
OFF
)
# FIXME Ideally this would be handled by presets and install rules, but for now
# consumers may override this to control the target location of cccl.c.experimental.stf.
set(
CCCL_C_EXPERIMENTAL_STF_LIBRARY_OUTPUT_DIRECTORY
""
CACHE PATH
"Override output directory for the cccl.c.experimental.stf library"
)
mark_as_advanced(CCCL_C_EXPERIMENTAL_STF_LIBRARY_OUTPUT_DIRECTORY)
file(
GLOB_RECURSE srcs
RELATIVE "${CMAKE_CURRENT_LIST_DIR}"
CONFIGURE_DEPENDS
"src/*.cu"
"src/*.cuh"
)
cccl_get_cudatoolkit()
cccl_get_cudax()
add_library(cccl.c.experimental.stf SHARED ${srcs})
set_property(
TARGET cccl.c.experimental.stf
PROPERTY POSITION_INDEPENDENT_CODE ON
)
cccl_configure_target(cccl.c.experimental.stf DIALECT 20)
# Override the properties set by cccl_configure_target:
if (CCCL_C_EXPERIMENTAL_STF_LIBRARY_OUTPUT_DIRECTORY)
set_target_properties(
cccl.c.experimental.stf
PROPERTIES
LIBRARY_OUTPUT_DIRECTORY
"${CCCL_C_EXPERIMENTAL_STF_LIBRARY_OUTPUT_DIRECTORY}"
ARCHIVE_OUTPUT_DIRECTORY
"${CCCL_C_EXPERIMENTAL_STF_LIBRARY_OUTPUT_DIRECTORY}"
)
endif()
set_target_properties(
cccl.c.experimental.stf
PROPERTIES CUDA_RUNTIME_LIBRARY STATIC
)
target_compile_definitions(cccl.c.experimental.stf PUBLIC CCCL_C_EXPERIMENTAL=1)
target_link_libraries(
cccl.c.experimental.stf
PRIVATE #
CUDA::cudart_static
CUDA::cuda_driver
cudax::cudax
)
target_compile_options(
cccl.c.experimental.stf
PRIVATE #
$<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:--expt-relaxed-constexpr>
$<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:--extended-lambda>
)
target_include_directories(
cccl.c.experimental.stf
PUBLIC "include"
PRIVATE "src"
)
if (CCCL_C_EXPERIMENTAL_STF_ENABLE_TESTING)
add_subdirectory(test)
endif()

2261
cccl_upstream/c/experimental/stf/include/cccl/c/experimental/stf/stf.h Normal file
View File

File diff suppressed because it is too large Load Diff

1938
cccl_upstream/c/experimental/stf/src/stf.cu Normal file
View File

File diff suppressed because it is too large Load Diff

44
cccl_upstream/c/experimental/stf/test/CMakeLists.txt Normal file
View File

@@ -0,0 +1,44 @@
cccl_get_c2h()
function(cccl_c_experimental_stf_add_test target_name_var source)
string(
REGEX REPLACE
"test_([^.]*)"
"cccl.c.experimental.stf.test.\\1"
target_name
"${source}"
)
set(target_name_var ${target_name} PARENT_SCOPE)
cccl_add_executable(
${target_name}
ADD_CTEST
NO_METATARGETS
DIALECT 20
SOURCES "${source}"
)
set_target_properties(${target_name} PROPERTIES CUDA_RUNTIME_LIBRARY STATIC)
target_link_libraries(
${target_name}
PRIVATE
cccl.compiler_interface
cccl.c.experimental.stf
CUDA::cudart_static
CUDA::nvrtc
cccl.c2h.main
CUDA::cuda_driver
)
endfunction()
file(
GLOB test_srcs
RELATIVE "${CMAKE_CURRENT_LIST_DIR}"
CONFIGURE_DEPENDS
*.cu
*.cpp
)
foreach (test_src IN LISTS test_srcs)
cccl_c_experimental_stf_add_test(test_target "${test_src}")
endforeach()

163
cccl_upstream/c/experimental/stf/test/test_allocate_nd.cpp Normal file
View File

@@ -0,0 +1,163 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cstdint>
#include <vector>
#include <cuda_runtime.h>
#include <c2h/catch2_test_helper.h>
#include <cccl/c/experimental/stf/stf.h>
namespace
{
inline constexpr uint64_t one_mib = 1024 * 1024;
stf_exec_place_handle make_dev0_grid(size_t nplaces)
{
std::vector<stf_exec_place_handle> places(nplaces);
for (auto& place : places)
{
place = stf_exec_place_device(0);
REQUIRE(place != nullptr);
}
stf_exec_place_handle const grid = stf_exec_place_grid_create(places.data(), nplaces, nullptr);
REQUIRE(grid != nullptr);
for (const auto& place : places)
{
stf_exec_place_destroy(place);
}
return grid;
}
void check_device_round_trip(void* ptr, uint64_t n)
{
const std::vector<int> host(n, 42);
REQUIRE(cudaMemcpy(ptr, host.data(), n * sizeof(int), cudaMemcpyHostToDevice) == cudaSuccess);
std::vector<int> back(n, 0);
REQUIRE(cudaMemcpy(back.data(), ptr, n * sizeof(int), cudaMemcpyDeviceToHost) == cudaSuccess);
REQUIRE(back[0] == 42);
REQUIRE(back[n - 1] == 42);
}
} // namespace
C2H_TEST("shaped allocation on an ordinary data place", "[places][allocate]")
{
constexpr uint64_t n = one_mib; // ints
constexpr stf_dim4 dims{n, 1, 1, 1};
stf_data_place_handle const dp = stf_data_place_device(0);
REQUIRE(dp != nullptr);
// On a non-composite place the geometry degenerates to a byte count
void* const ptr = stf_data_place_allocate_nd(dp, &dims, sizeof(int), nullptr);
REQUIRE(ptr != nullptr);
check_device_round_trip(ptr, n);
stf_data_place_deallocate(dp, ptr, n * sizeof(int), nullptr);
stf_data_place_destroy(dp);
}
C2H_TEST("shaped allocation on composite data places", "[places][allocate]")
{
stf_exec_place_handle const grid = make_dev0_grid(2);
constexpr uint64_t n = one_mib; // ints
constexpr stf_dim4 dims{n, 1, 1, 1};
stf_data_place_handle const dp = stf_data_place_composite(grid, stf_partition_fn_blocked(0));
REQUIRE(dp != nullptr);
// A byte count alone cannot carry the tensor geometry: must fail cleanly
void* const bad = stf_data_place_allocate(dp, static_cast<ptrdiff_t>(n * sizeof(int)), nullptr);
REQUIRE(bad == nullptr);
void* const ptr = stf_data_place_allocate_nd(dp, &dims, sizeof(int), nullptr);
REQUIRE(ptr != nullptr);
// Memory must be usable from the device
check_device_round_trip(ptr, n);
stf_data_place_deallocate(dp, ptr, n * sizeof(int), nullptr);
stf_data_place_destroy(dp);
// Same flow through the native cyclic partition function
stf_data_place_handle const dpc = stf_data_place_composite(grid, stf_partition_fn_cyclic());
REQUIRE(dpc != nullptr);
void* const ptr2 = stf_data_place_allocate_nd(dpc, &dims, sizeof(int), nullptr);
REQUIRE(ptr2 != nullptr);
check_device_round_trip(ptr2, n);
stf_data_place_deallocate(dpc, ptr2, n * sizeof(int), nullptr);
stf_data_place_destroy(dpc);
stf_exec_place_destroy(grid);
}
C2H_TEST("blocked partition function covers every dimension selector", "[places][allocate]")
{
stf_exec_place_handle const grid = make_dev0_grid(2);
// 64 * 64 * 16 * 4 ints = 1 MiB: every dimension is divisible by the grid
constexpr stf_dim4 dims{64, 64, 16, 4};
constexpr uint64_t n = dims.x * dims.y * dims.z * dims.t;
// Dimensions 0-3 select that axis; out-of-range values (like -1) select the
// highest axis whose extent is greater than one. All must yield a usable
// native mapper.
for (const int dim : {0, 1, 2, 3, -1, 4})
{
const stf_get_executor_fn mapper = stf_partition_fn_blocked(dim);
REQUIRE(mapper != nullptr);
stf_data_place_handle const dp = stf_data_place_composite(grid, mapper);
REQUIRE(dp != nullptr);
void* const ptr = stf_data_place_allocate_nd(dp, &dims, sizeof(int), nullptr);
REQUIRE(ptr != nullptr);
check_device_round_trip(ptr, n);
stf_data_place_deallocate(dp, ptr, n * sizeof(int), nullptr);
stf_data_place_destroy(dp);
}
stf_exec_place_destroy(grid);
}
C2H_TEST("shaped allocation rejects overflowing geometries", "[places][allocate]")
{
// (2^64-1)^2 wraps to 1: an unchecked size computation would hand back a
// one-byte allocation for an astronomically large tensor
constexpr stf_dim4 huge{UINT64_MAX, UINT64_MAX, 1, 1};
stf_data_place_handle const dp = stf_data_place_device(0);
REQUIRE(dp != nullptr);
REQUIRE(stf_data_place_allocate_nd(dp, &huge, 1, nullptr) == nullptr);
// elemsize participates in the product too
constexpr stf_dim4 max_1d{UINT64_MAX, 1, 1, 1};
REQUIRE(stf_data_place_allocate_nd(dp, &max_1d, 2, nullptr) == nullptr);
// A representable product that exceeds PTRDIFF_MAX must also be rejected
constexpr stf_dim4 above_ptrdiff{uint64_t{1} << 62, 2, 1, 1};
REQUIRE(stf_data_place_allocate_nd(dp, &above_ptrdiff, 1, nullptr) == nullptr);
stf_data_place_destroy(dp);
// On a composite place the wrapped geometry used to reach the blocked
// partitioner with a zero part_size and kill the process with SIGFPE
stf_exec_place_handle const grid = make_dev0_grid(2);
stf_data_place_handle const dpc = stf_data_place_composite(grid, stf_partition_fn_blocked(1));
REQUIRE(dpc != nullptr);
REQUIRE(stf_data_place_allocate_nd(dpc, &huge, 1, nullptr) == nullptr);
stf_data_place_destroy(dpc);
stf_exec_place_destroy(grid);
}

208
cccl_upstream/c/experimental/stf/test/test_async_resources_handle.cu Normal file
View File

@@ -0,0 +1,208 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
// Focused tests for stf_async_resources_create/destroy() exercised through
// stf_ctx_create_ex(). They cover the contract documented in stf.h:
//
// * A shared stf_async_resources_handle can be reused across multiple
// contexts created via stf_ctx_create_ex().
// * When the contexts are created with `has_stream = 1`, stf_ctx_finalize()
// is non-blocking: the caller must cudaStreamSynchronize(user_stream)
// before destroying the shared handle.
//
// Both backends (STF_BACKEND_STREAM and STF_BACKEND_GRAPH) are exercised
// because the graph backend additionally benefits from the handle's
// executable-graph cache.
#include <cuda_runtime.h>
#include <c2h/catch2_test_helper.h>
#include <cccl/c/experimental/stf/stf.h>
// A device sink that is written but never read. Publishing the busy-loop
// result here gives the loop an observable side effect, so the compiler
// cannot optimize it away, without perturbing the result buffer.
__device__ unsigned g_busy_sink;
// Writes `value` into every slot of `arr`. The inner busy loop widens the
// kernel window so a missing chain dependency between back-to-back contexts
// becomes observable: a slow ctx1 kernel must finish before ctx2's kernel
// commits its value.
__global__ void slow_set_kernel(int* arr, int n, int value, int iters)
{
const int tid = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
if (tid >= n)
{
return;
}
// Busy loop to keep the kernel resident on the SM for a while. `acc` is
// unsigned so the accumulation wraps with well-defined behavior.
unsigned acc = 0;
for (int i = 0; i < iters; ++i)
{
acc += (static_cast<unsigned>(i) * 1103515245u + 12345u) & 0x7fffffffu;
}
// Publish `acc` via an atomic: an observable, race-free side effect that
// keeps the loop alive while the stored result stays exactly `value`.
atomicAdd(&g_busy_sink, acc);
arr[tid] = value;
}
namespace
{
// Submit one slow_set kernel into `ctx`, writing `value` everywhere in
// `d_arr`. Use stf_cuda_kernel_* instead of the generic task stream API so
// this helper is valid for both stream and graph backends.
void submit_set_kernel(stf_ctx_handle ctx, int* d_arr, int n, int value, int iters)
{
int dev_id = 0;
REQUIRE(cudaGetDevice(&dev_id) == cudaSuccess);
stf_data_place_handle dev_place = stf_data_place_device(dev_id);
stf_logical_data_handle lD = stf_logical_data_with_place(ctx, d_arr, n * sizeof(int), dev_place);
REQUIRE(lD != nullptr);
stf_data_place_destroy(dev_place);
stf_logical_data_set_symbol(lD, "device_buffer");
stf_cuda_kernel_handle k = stf_cuda_kernel_create(ctx);
REQUIRE(k != nullptr);
stf_cuda_kernel_set_symbol(k, "slow_set");
stf_cuda_kernel_add_dep(k, lD, STF_RW);
stf_cuda_kernel_start(k);
int* arg_ptr = static_cast<int*>(stf_cuda_kernel_get_arg(k, 0));
REQUIRE(arg_ptr == d_arr);
const int threads = 128;
const int blocks = (n + threads - 1) / threads;
const void* args[4] = {&arg_ptr, &n, &value, &iters};
cudaError_t err =
stf_cuda_kernel_add_desc(k, reinterpret_cast<void*>(slow_set_kernel), dim3(blocks), dim3(threads), 0, 4, args);
REQUIRE(err == cudaSuccess);
stf_cuda_kernel_end(k);
stf_cuda_kernel_destroy(k);
stf_logical_data_destroy(lD);
}
// Run one ctx (created via stf_ctx_create_ex with a caller-provided stream
// and a shared async_resources handle) that issues a single slow_set kernel.
void run_ctx_with_handle(
stf_backend_kind backend, cudaStream_t s, stf_async_resources_handle h, int* d_arr, int N, int value, int iters)
{
stf_ctx_options opts{};
opts.backend = backend;
opts.has_stream = 1;
opts.stream = s;
opts.handle = h;
stf_ctx_handle ctx = stf_ctx_create_ex(&opts);
REQUIRE(ctx != nullptr);
submit_set_kernel(ctx, d_arr, N, value, iters);
// Non-blocking: this enqueues the remaining work and the resource-release
// callback on `s`; it does not synchronize `s`.
stf_ctx_finalize(ctx);
}
// Run a back-to-back ordering experiment: two contexts share a handle and a
// caller stream, write distinct values, and the final buffer must reflect
// the second context's value (ctx2-after-ctx1 ordering via the caller
// stream). Iterating amplifies any missed dependency.
void check_back_to_back_ordering(stf_backend_kind backend)
{
constexpr int N = 1 << 14;
constexpr int ITERS = 1 << 18;
cudaStream_t s{};
REQUIRE(cudaStreamCreate(&s) == cudaSuccess);
int* d_arr = nullptr;
REQUIRE(cudaMalloc(&d_arr, N * sizeof(int)) == cudaSuccess);
REQUIRE(cudaMemsetAsync(d_arr, 0, N * sizeof(int), s) == cudaSuccess);
stf_async_resources_handle h = stf_async_resources_create();
REQUIRE(h != nullptr);
for (int iter = 0; iter < 20; ++iter)
{
run_ctx_with_handle(backend, s, h, d_arr, N, /*value=*/1, ITERS);
run_ctx_with_handle(backend, s, h, d_arr, N, /*value=*/2, ITERS);
REQUIRE(cudaStreamSynchronize(s) == cudaSuccess);
int h_arr[16]{};
REQUIRE(cudaMemcpy(h_arr, d_arr, sizeof(h_arr), cudaMemcpyDeviceToHost) == cudaSuccess);
for (int i = 0; i < static_cast<int>(sizeof(h_arr) / sizeof(int)); ++i)
{
INFO("iter=" << iter << " i=" << i << " value=" << h_arr[i]);
REQUIRE(h_arr[i] == 2);
}
}
// Required before destroying `h`: stf_ctx_finalize() left resource-release
// callbacks enqueued on `s`. The destroy call tears down the underlying
// CUDA resources synchronously.
REQUIRE(cudaStreamSynchronize(s) == cudaSuccess);
stf_async_resources_destroy(h);
REQUIRE(cudaFree(d_arr) == cudaSuccess);
REQUIRE(cudaStreamDestroy(s) == cudaSuccess);
}
} // namespace
C2H_TEST("stf_async_resources_handle: shared across back-to-back stream contexts on user stream",
"[context][stream][async_resources_handle]")
{
check_back_to_back_ordering(STF_BACKEND_STREAM);
}
C2H_TEST("stf_async_resources_handle: shared across back-to-back graph contexts on user stream",
"[context][graph][async_resources_handle]")
{
check_back_to_back_ordering(STF_BACKEND_GRAPH);
}
// Smoke check of the handle's lifetime API independent of any context:
// * NULL is a no-op for stf_async_resources_destroy().
// * Create/destroy without ever attaching the handle to a context works.
// * Destroying the handle before having submitted any work via a context
// (only after a context was created and finalized without tasks) is
// safe.
C2H_TEST("stf_async_resources_handle: lifetime smoke (no work, NULL destroy)",
"[context][async_resources_handle][lifetime]")
{
stf_async_resources_destroy(nullptr);
stf_async_resources_handle h = stf_async_resources_create();
REQUIRE(h != nullptr);
stf_async_resources_destroy(h);
// Empty stream context bound to a user stream + handle, no submitted work.
cudaStream_t s{};
REQUIRE(cudaStreamCreate(&s) == cudaSuccess);
stf_async_resources_handle h2 = stf_async_resources_create();
REQUIRE(h2 != nullptr);
{
stf_ctx_options opts{};
opts.backend = STF_BACKEND_STREAM;
opts.has_stream = 1;
opts.stream = s;
opts.handle = h2;
stf_ctx_handle ctx = stf_ctx_create_ex(&opts);
REQUIRE(ctx != nullptr);
stf_ctx_finalize(ctx);
}
// Even without submitted tasks the finalize is non-blocking when the
// context was created with `has_stream = 1`. Synchronize before destroy.
REQUIRE(cudaStreamSynchronize(s) == cudaSuccess);
stf_async_resources_destroy(h2);
REQUIRE(cudaStreamDestroy(s) == cudaSuccess);
}

92
cccl_upstream/c/experimental/stf/test/test_ctx.cpp Normal file
View File

@@ -0,0 +1,92 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda_runtime.h>
#include <c2h/catch2_test_helper.h>
#include <cccl/c/experimental/stf/stf.h>
C2H_TEST("basic stf context", "[context]")
{
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
stf_ctx_finalize(ctx);
}
C2H_TEST("stf_ctx_wait reads data without finalizing", "[context]")
{
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
int h_value = 0;
stf_logical_data_handle lVal = stf_logical_data(ctx, &h_value, sizeof(int));
REQUIRE(lVal != nullptr);
stf_logical_data_set_symbol(lVal, "val");
int src_val = 42;
stf_host_launch_handle h = stf_host_launch_create(ctx);
REQUIRE(h != nullptr);
stf_host_launch_set_symbol(h, "set42");
stf_host_launch_add_dep(h, lVal, STF_WRITE);
stf_host_launch_set_user_data(h, &src_val, sizeof(int), nullptr);
stf_host_launch_submit(h, [](stf_host_launch_deps_handle deps) {
int* data = (int*) stf_host_launch_deps_get(deps, 0);
int* src = (int*) stf_host_launch_deps_get_user_data(deps);
data[0] = *src;
});
stf_host_launch_destroy(h);
int result = 0;
int rc = stf_ctx_wait(ctx, lVal, &result, sizeof(int));
REQUIRE(rc == 0);
REQUIRE(result == 42);
// The context remains usable after waiting.
src_val = 99;
stf_host_launch_handle h2 = stf_host_launch_create(ctx);
REQUIRE(h2 != nullptr);
stf_host_launch_set_symbol(h2, "set99");
stf_host_launch_add_dep(h2, lVal, STF_WRITE);
stf_host_launch_set_user_data(h2, &src_val, sizeof(int), nullptr);
stf_host_launch_submit(h2, [](stf_host_launch_deps_handle deps) {
int* data = (int*) stf_host_launch_deps_get(deps, 0);
int* src = (int*) stf_host_launch_deps_get_user_data(deps);
data[0] = *src;
});
stf_host_launch_destroy(h2);
result = 0;
rc = stf_ctx_wait(ctx, lVal, &result, sizeof(int));
REQUIRE(rc == 0);
REQUIRE(result == 99);
stf_logical_data_destroy(lVal);
stf_ctx_finalize(ctx);
}
C2H_TEST("stf_ctx_wait rejects invalid arguments", "[context]")
{
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
int h_value = 0;
stf_logical_data_handle lVal = stf_logical_data(ctx, &h_value, sizeof(int));
REQUIRE(lVal != nullptr);
int result = 0;
REQUIRE(stf_ctx_wait(nullptr, lVal, &result, sizeof(int)) != 0);
REQUIRE(stf_ctx_wait(ctx, nullptr, &result, sizeof(int)) != 0);
REQUIRE(stf_ctx_wait(ctx, lVal, nullptr, sizeof(int)) != 0);
stf_logical_data_destroy(lVal);
stf_ctx_finalize(ctx);
}

89
cccl_upstream/c/experimental/stf/test/test_cuda_kernel.cu Normal file
View File

@@ -0,0 +1,89 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <vector>
#include <cuda_runtime.h>
#include <c2h/catch2_test_helper.h>
#include <cccl/c/experimental/stf/stf.h>
__global__ void axpy(int cnt, double a, const double* x, double* y)
{
int tid = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
int nthreads = static_cast<int>(gridDim.x * blockDim.x);
for (int i = tid; i < cnt; i += nthreads)
{
y[i] += a * x[i];
}
}
double X0(int i)
{
return sin(static_cast<double>(i));
}
double Y0(int i)
{
return cos((double) i);
}
C2H_TEST("axpy with stf cuda_kernel", "[cuda_kernel]")
{
size_t N = 1000000;
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
std::vector<double> X(N);
std::vector<double> Y(N);
for (size_t i = 0; i < N; i++)
{
X[i] = X0(static_cast<int>(i));
Y[i] = Y0(static_cast<int>(i));
}
const double alpha = 3.14;
stf_logical_data_handle lX = stf_logical_data(ctx, X.data(), N * sizeof(double));
stf_logical_data_handle lY = stf_logical_data(ctx, Y.data(), N * sizeof(double));
REQUIRE(lX != nullptr);
REQUIRE(lY != nullptr);
stf_logical_data_set_symbol(lX, "X");
stf_logical_data_set_symbol(lY, "Y");
stf_cuda_kernel_handle k = stf_cuda_kernel_create(ctx);
REQUIRE(k != nullptr);
stf_cuda_kernel_set_symbol(k, "axpy");
stf_cuda_kernel_add_dep(k, lX, STF_READ);
stf_cuda_kernel_add_dep(k, lY, STF_RW);
stf_cuda_kernel_start(k);
double* dX = (double*) stf_cuda_kernel_get_arg(k, 0);
double* dY = (double*) stf_cuda_kernel_get_arg(k, 1);
const void* args[4] = {&N, &alpha, &dX, &dY};
cudaError_t err = stf_cuda_kernel_add_desc(k, (void*) axpy, 2, 4, 0, 4, args);
REQUIRE(err == cudaSuccess);
stf_cuda_kernel_end(k);
stf_cuda_kernel_destroy(k);
stf_logical_data_destroy(lX);
stf_logical_data_destroy(lY);
stf_ctx_finalize(ctx);
for (size_t i = 0; i < N; i++)
{
assert(fabs(Y[i] - (Y0(i) + alpha * X0(i))) < 0.0001);
assert(fabs(X[i] - X0(i)) < 0.0001);
}
}

263
cccl_upstream/c/experimental/stf/test/test_host_launch.cu Normal file
View File

@@ -0,0 +1,263 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cmath>
#include <cuda_runtime.h>
#include <c2h/catch2_test_helper.h>
#include <cccl/c/experimental/stf/stf.h>
__global__ void fill_kernel(int cnt, double* data, double value)
{
int tid = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
int nthreads = static_cast<int>(gridDim.x * blockDim.x);
for (int i = tid; i < cnt; i += nthreads)
{
data[i] = value + i;
}
}
struct verify_args
{
size_t N;
bool* passed;
};
static void verify_callback(stf_host_launch_deps_handle deps)
{
auto* v = static_cast<verify_args*>(stf_host_launch_deps_get_user_data(deps));
if (stf_host_launch_deps_size(deps) != 1)
{
*v->passed = false;
return;
}
if (stf_host_launch_deps_get_size(deps, 0) != v->N * sizeof(double))
{
*v->passed = false;
return;
}
auto* data = static_cast<double*>(stf_host_launch_deps_get(deps, 0));
for (size_t i = 0; i < v->N; i++)
{
if (fabs(data[i] - (42.0 + static_cast<double>(i))) > 1e-10)
{
*v->passed = false;
return;
}
}
*v->passed = true;
}
C2H_TEST("host_launch with stream context", "[host_launch]")
{
const size_t N = 1024;
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
double* host_data;
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
for (size_t i = 0; i < N; i++)
{
host_data[i] = 0.0;
}
stf_logical_data_handle lData = stf_logical_data(ctx, host_data, N * sizeof(double));
REQUIRE(lData != nullptr);
stf_logical_data_set_symbol(lData, "data");
// Fill data via a kernel task
stf_task_handle t = stf_task_create(ctx);
REQUIRE(t != nullptr);
stf_task_set_symbol(t, "fill");
stf_task_add_dep(t, lData, STF_WRITE);
stf_task_start(t);
double* dData = (double*) stf_task_get(t, 0);
fill_kernel<<<2, 128, 0, (cudaStream_t) stf_task_get_custream(t)>>>((int) N, dData, 42.0);
stf_task_end(t);
stf_task_destroy(t);
// Use host_launch to verify data on the host
bool passed = false;
verify_args vargs{N, &passed};
stf_host_launch_handle h = stf_host_launch_create(ctx);
REQUIRE(h != nullptr);
stf_host_launch_set_symbol(h, "verify");
stf_host_launch_add_dep(h, lData, STF_READ);
stf_host_launch_set_user_data(h, &vargs, sizeof(vargs), nullptr);
stf_host_launch_submit(h, verify_callback);
stf_host_launch_destroy(h);
stf_logical_data_destroy(lData);
stf_ctx_finalize(ctx);
REQUIRE(passed);
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
}
C2H_TEST("host_launch with graph context", "[host_launch]")
{
const size_t N = 1024;
stf_ctx_handle ctx = stf_ctx_create_graph();
REQUIRE(ctx != nullptr);
double* host_data;
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
for (size_t i = 0; i < N; i++)
{
host_data[i] = 0.0;
}
stf_logical_data_handle lData = stf_logical_data(ctx, host_data, N * sizeof(double));
REQUIRE(lData != nullptr);
stf_logical_data_set_symbol(lData, "data");
// Fill data via a generic task with stream capture
stf_task_handle t = stf_task_create(ctx);
REQUIRE(t != nullptr);
stf_task_set_symbol(t, "fill");
stf_task_add_dep(t, lData, STF_WRITE);
stf_task_enable_capture(t);
stf_task_start(t);
double* dData = (double*) stf_task_get(t, 0);
cudaStream_t stream = (cudaStream_t) stf_task_get_custream(t);
fill_kernel<<<2, 128, 0, stream>>>((int) N, dData, 42.0);
stf_task_end(t);
stf_task_destroy(t);
// Use host_launch to verify data on the host
bool passed = false;
verify_args vargs{N, &passed};
stf_host_launch_handle h = stf_host_launch_create(ctx);
REQUIRE(h != nullptr);
stf_host_launch_set_symbol(h, "verify");
stf_host_launch_add_dep(h, lData, STF_READ);
stf_host_launch_set_user_data(h, &vargs, sizeof(vargs), nullptr);
stf_host_launch_submit(h, verify_callback);
stf_host_launch_destroy(h);
stf_logical_data_destroy(lData);
stf_ctx_finalize(ctx);
REQUIRE(passed);
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
}
C2H_TEST("host_launch with stackable context", "[host_launch][stackable]")
{
const size_t N = 1024;
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
double* host_data;
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
for (size_t i = 0; i < N; i++)
{
host_data[i] = 0.0;
}
stf_logical_data_handle lData = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
REQUIRE(lData != nullptr);
stf_stackable_logical_data_set_symbol(lData, "data");
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_task_set_symbol(t, "fill");
stf_stackable_task_add_dep(ctx, t, lData, STF_WRITE);
stf_task_start(t);
double* dData = (double*) stf_task_get(t, 0);
fill_kernel<<<2, 128, 0, (cudaStream_t) stf_task_get_custream(t)>>>((int) N, dData, 42.0);
stf_task_end(t);
stf_task_destroy(t);
bool passed = false;
verify_args vargs{N, &passed};
stf_host_launch_handle h = stf_stackable_host_launch_create(ctx);
REQUIRE(h != nullptr);
stf_host_launch_set_symbol(h, "verify");
stf_stackable_host_launch_add_dep(ctx, h, lData, STF_READ);
stf_host_launch_set_user_data(h, &vargs, sizeof(vargs), nullptr);
stf_stackable_host_launch_submit(h, verify_callback);
stf_stackable_host_launch_destroy(h);
stf_stackable_logical_data_destroy(lData);
stf_stackable_ctx_finalize(ctx);
REQUIRE(passed);
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
}
C2H_TEST("host_launch inside a stackable nested graph scope", "[host_launch][stackable]")
{
const size_t N = 1024;
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
double* host_data;
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
for (size_t i = 0; i < N; i++)
{
host_data[i] = 0.0;
}
stf_logical_data_handle lData = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
REQUIRE(lData != nullptr);
// Push a nested graph scope and run both the producer task and the host_launch
// verifier inside it. The data auto-pushes from root to the nested scope.
stf_stackable_push_graph(ctx);
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, lData, STF_WRITE);
// Inside a nested graph scope the task is captured into the child graph, so we
// must enable capture to obtain the graph's capture stream. Otherwise
// stf_task_get_custream() returns a null/uninitialized stream and the kernel
// would run outside the STF graph, racing the host verifier below.
stf_task_enable_capture(t);
stf_task_start(t);
double* dData = (double*) stf_task_get(t, 0);
fill_kernel<<<2, 128, 0, (cudaStream_t) stf_task_get_custream(t)>>>((int) N, dData, 42.0);
stf_task_end(t);
stf_task_destroy(t);
bool passed = false;
verify_args vargs{N, &passed};
stf_host_launch_handle h = stf_stackable_host_launch_create(ctx);
REQUIRE(h != nullptr);
stf_stackable_host_launch_add_dep(ctx, h, lData, STF_READ);
stf_host_launch_set_user_data(h, &vargs, sizeof(vargs), nullptr);
stf_stackable_host_launch_submit(h, verify_callback);
stf_stackable_host_launch_destroy(h);
stf_stackable_pop(ctx);
stf_stackable_logical_data_destroy(lData);
stf_stackable_ctx_finalize(ctx);
REQUIRE(passed);
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
}

37
cccl_upstream/c/experimental/stf/test/test_logical_data.cpp Normal file
View File

@@ -0,0 +1,37 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <vector>
#include <cuda_runtime.h>
#include <c2h/catch2_test_helper.h>
#include <cccl/c/experimental/stf/stf.h>
C2H_TEST("basic stf logical_data", "[logical_data]")
{
size_t N = 1000000;
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
std::vector<float> A(N);
std::vector<float> B(N);
stf_logical_data_handle lA = stf_logical_data(ctx, A.data(), N * sizeof(float));
stf_logical_data_handle lB = stf_logical_data(ctx, B.data(), N * sizeof(float));
REQUIRE(lA != nullptr);
REQUIRE(lB != nullptr);
stf_logical_data_destroy(lA);
stf_logical_data_destroy(lB);
stf_ctx_finalize(ctx);
}

161
cccl_upstream/c/experimental/stf/test/test_logical_data_with_place.cu Normal file
View File

@@ -0,0 +1,161 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
//
// Unit tests for stf_logical_data_with_place(): logical data with explicit
// data place (host, pinned host, device).
//
//===----------------------------------------------------------------------===//
#include <memory>
#include <vector>
#include <cuda_runtime.h>
#include <c2h/catch2_test_helper.h>
#include <cccl/c/experimental/stf/stf.h>
__global__ void scale_inplace(int n, float* data, float factor)
{
int i = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
if (i < n)
{
data[i] *= factor;
}
}
C2H_TEST("stf_logical_data_with_place - host place (malloc)", "[logical_data_with_place]")
{
size_t N = 1024;
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
std::vector<float> A(N);
for (size_t i = 0; i < N; ++i)
{
A[i] = static_cast<float>(i);
}
stf_data_place_handle host_place = stf_data_place_host();
stf_logical_data_handle lA = stf_logical_data_with_place(ctx, A.data(), N * sizeof(float), host_place);
REQUIRE(lA != nullptr);
stf_data_place_destroy(host_place);
stf_task_handle t = stf_task_create(ctx);
REQUIRE(t != nullptr);
stf_task_add_dep(t, lA, STF_RW);
stf_task_start(t);
stf_task_end(t);
stf_task_destroy(t);
stf_logical_data_destroy(lA);
stf_ctx_finalize(ctx);
for (size_t i = 0; i < N; ++i)
{
REQUIRE(A[i] == static_cast<float>(i));
}
}
C2H_TEST("stf_logical_data_with_place - host place (pinned memory)", "[logical_data_with_place]")
{
size_t N = 1024;
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
float* A = nullptr;
cudaError_t err = cudaMallocHost(&A, N * sizeof(float));
REQUIRE(err == cudaSuccess);
for (size_t i = 0; i < N; ++i)
{
A[i] = static_cast<float>(i);
}
stf_data_place_handle host_place = stf_data_place_host();
stf_logical_data_handle lA = stf_logical_data_with_place(ctx, A, N * sizeof(float), host_place);
REQUIRE(lA != nullptr);
stf_data_place_destroy(host_place);
stf_task_handle t = stf_task_create(ctx);
REQUIRE(t != nullptr);
stf_task_add_dep(t, lA, STF_RW);
stf_task_start(t);
stf_task_end(t);
stf_task_destroy(t);
stf_logical_data_destroy(lA);
stf_ctx_finalize(ctx);
for (size_t i = 0; i < N; ++i)
{
REQUIRE(A[i] == static_cast<float>(i));
}
REQUIRE(cudaFreeHost(A) == cudaSuccess);
}
C2H_TEST("stf_logical_data_with_place - device place (data on current device)", "[logical_data_with_place]")
{
size_t N = 1024;
const float factor = 2.0f;
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
float* d_raw = nullptr;
cudaError_t err = cudaMalloc(&d_raw, N * sizeof(float));
REQUIRE(err == cudaSuccess);
std::unique_ptr<void, decltype(&cudaFree)> d_data_owner(d_raw, cudaFree);
float* d_data = static_cast<float*>(d_data_owner.get());
std::vector<float> h_init(N);
for (size_t i = 0; i < N; ++i)
{
h_init[i] = static_cast<float>(i);
}
err = cudaMemcpy(d_data, h_init.data(), N * sizeof(float), cudaMemcpyHostToDevice);
REQUIRE(err == cudaSuccess);
stf_data_place_handle dev_place = stf_data_place_device(0);
stf_logical_data_handle lD = stf_logical_data_with_place(ctx, d_data, N * sizeof(float), dev_place);
REQUIRE(lD != nullptr);
stf_data_place_destroy(dev_place);
stf_logical_data_set_symbol(lD, "device_buf");
stf_cuda_kernel_handle k = stf_cuda_kernel_create(ctx);
REQUIRE(k != nullptr);
stf_cuda_kernel_set_symbol(k, "scale_inplace");
stf_cuda_kernel_add_dep(k, lD, STF_RW);
stf_cuda_kernel_start(k);
float* arg_ptr = static_cast<float*>(stf_cuda_kernel_get_arg(k, 0));
REQUIRE(arg_ptr == d_data);
int n = static_cast<int>(N);
const void* args[3] = {&n, &arg_ptr, &factor};
dim3 grid(4);
dim3 block(256);
err = stf_cuda_kernel_add_desc(k, reinterpret_cast<void*>(scale_inplace), grid, block, 0, 3, args);
REQUIRE(err == cudaSuccess);
stf_cuda_kernel_end(k);
stf_cuda_kernel_destroy(k);
stf_logical_data_destroy(lD);
stf_ctx_finalize(ctx);
// Copy back and verify: should be i * factor
std::vector<float> h_result(N);
err = cudaMemcpy(h_result.data(), d_data, N * sizeof(float), cudaMemcpyDeviceToHost);
REQUIRE(err == cudaSuccess);
for (size_t i = 0; i < N; ++i)
{
REQUIRE(h_result[i] == static_cast<float>(i) * factor);
}
}

679
cccl_upstream/c/experimental/stf/test/test_places.cpp Normal file
View File

@@ -0,0 +1,679 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/__cmath/ceil_div.h>
#include <string>
#include <vector>
#include <cuda_runtime.h>
#include <c2h/catch2_test_helper.h>
#include <cccl/c/experimental/stf/stf.h>
// Blocked partition along first dimension: maps data coordinates to grid position.
// Used to exercise composite data place with a grid of execution places.
static void blocked_mapper_1d(stf_pos4* result, stf_pos4 data_coords, stf_dim4 data_dims, stf_dim4 grid_dims)
{
uint64_t extent = data_dims.x;
uint64_t nplaces = grid_dims.x;
uint64_t part_size = ::cuda::ceil_div(extent, nplaces);
if (part_size == 0)
{
part_size = 1;
}
int64_t c = static_cast<int64_t>(data_coords.x);
int64_t place_x = c / static_cast<int64_t>(part_size);
if (place_x >= static_cast<int64_t>(nplaces))
{
place_x = static_cast<int64_t>(nplaces) - 1;
}
result->x = place_x;
result->y = 0;
result->z = 0;
result->t = 0;
}
C2H_TEST("exec place from an externally-owned CUDA context", "[task][places][cuda_context]")
{
constexpr size_t element_count{1024};
// Wrap the primary context of device 0 as an exec place
CUdevice dev = 0;
REQUIRE(cuDeviceGet(&dev, 0) == CUDA_SUCCESS);
CUcontext primary_ctx = nullptr;
REQUIRE(cuDevicePrimaryCtxRetain(&primary_ctx, dev) == CUDA_SUCCESS);
// Null context is rejected
REQUIRE(stf_exec_place_cuda_context(nullptr, 0) == nullptr);
// dev_id < 0 is derived from the context
const stf_exec_place_handle place_derived = stf_exec_place_cuda_context(primary_ctx, -1);
REQUIRE(place_derived != nullptr);
REQUIRE(stf_exec_place_is_device(place_derived) != 0);
stf_exec_place_destroy(place_derived);
const stf_exec_place_handle place = stf_exec_place_cuda_context(primary_ctx, 0);
REQUIRE(place != nullptr);
REQUIRE(stf_exec_place_is_device(place) != 0);
REQUIRE(stf_exec_place_is_host(place) == 0);
// Run a task on the place and fill the buffer through its stream
const stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
std::vector<float> x(element_count, 1.0f);
const stf_logical_data_handle logical_x = stf_logical_data(ctx, x.data(), element_count * sizeof(float));
REQUIRE(logical_x != nullptr);
const stf_task_handle task = stf_task_create(ctx);
REQUIRE(task != nullptr);
stf_task_set_exec_place(task, place);
stf_task_add_dep(task, logical_x, STF_RW);
stf_task_start(task);
const CUstream stream = stf_task_get_custream(task);
REQUIRE(stream != nullptr);
float* const device_x = static_cast<float*>(stf_task_get(task, 0));
REQUIRE(device_x != nullptr);
REQUIRE(cudaMemsetAsync(device_x, 0, element_count * sizeof(float), stream) == cudaSuccess);
stf_task_end(task);
stf_task_destroy(task);
stf_logical_data_destroy(logical_x);
stf_ctx_finalize(ctx);
for (size_t i = 0; i < element_count; i++)
{
REQUIRE(x[i] == 0.0f);
}
stf_exec_place_destroy(place);
REQUIRE(cuDevicePrimaryCtxRelease(dev) == CUDA_SUCCESS);
}
C2H_TEST("empty stf tasks", "[task]")
{
size_t N = 1000000;
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
std::vector<float> X(N);
std::vector<float> Y(N);
std::vector<float> Z(N);
stf_logical_data_handle lX = stf_logical_data(ctx, X.data(), N * sizeof(float));
stf_logical_data_handle lY = stf_logical_data(ctx, Y.data(), N * sizeof(float));
stf_logical_data_handle lZ = stf_logical_data(ctx, Z.data(), N * sizeof(float));
REQUIRE(lX != nullptr);
REQUIRE(lY != nullptr);
REQUIRE(lZ != nullptr);
stf_logical_data_set_symbol(lX, "X");
stf_logical_data_set_symbol(lY, "Y");
stf_logical_data_set_symbol(lZ, "Z");
stf_task_handle t1 = stf_task_create(ctx);
REQUIRE(t1 != nullptr);
stf_task_set_symbol(t1, "T1");
stf_task_add_dep(t1, lX, STF_RW);
stf_task_start(t1);
stf_task_end(t1);
stf_task_destroy(t1);
stf_task_handle t2 = stf_task_create(ctx);
REQUIRE(t2 != nullptr);
stf_task_set_symbol(t2, "T2");
stf_task_add_dep(t2, lX, STF_READ);
stf_task_add_dep(t2, lY, STF_RW);
stf_task_start(t2);
stf_task_end(t2);
stf_task_destroy(t2);
stf_task_handle t3 = stf_task_create(ctx);
REQUIRE(t3 != nullptr);
stf_task_set_symbol(t3, "T3");
stf_exec_place_handle e_place_dev0 = stf_exec_place_device(0);
stf_task_set_exec_place(t3, e_place_dev0);
stf_exec_place_destroy(e_place_dev0);
stf_task_add_dep(t3, lX, STF_READ);
stf_task_add_dep(t3, lZ, STF_RW);
stf_task_start(t3);
stf_task_end(t3);
stf_task_destroy(t3);
stf_task_handle t4 = stf_task_create(ctx);
REQUIRE(t4 != nullptr);
stf_task_set_symbol(t4, "T4");
stf_task_add_dep(t4, lY, STF_READ);
stf_data_place_handle d_place_dev0 = stf_data_place_device(0);
stf_task_add_dep_with_dplace(t4, lZ, STF_RW, d_place_dev0);
stf_data_place_destroy(d_place_dev0);
stf_task_start(t4);
stf_task_end(t4);
stf_task_destroy(t4);
stf_logical_data_destroy(lX);
stf_logical_data_destroy(lY);
stf_logical_data_destroy(lZ);
stf_ctx_finalize(ctx);
}
C2H_TEST("composite data place with grid of places (same device repeated)", "[task][places][composite]")
{
const size_t nplaces = 3;
stf_exec_place_handle places[3];
for (auto& place : places)
{
place = stf_exec_place_device(0);
}
stf_exec_place_handle grid = stf_exec_place_grid_create(places, nplaces, nullptr);
REQUIRE(grid != nullptr);
for (auto& place : places)
{
stf_exec_place_destroy(place);
}
stf_data_place_handle composite_dplace = stf_data_place_composite(grid, blocked_mapper_1d);
REQUIRE(composite_dplace != nullptr);
stf_exec_place_grid_destroy(grid);
size_t N = 1024;
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
std::vector<float> X(N);
for (size_t i = 0; i < N; ++i)
{
X[i] = static_cast<float>(i);
}
stf_logical_data_handle lX = stf_logical_data(ctx, X.data(), N * sizeof(float));
REQUIRE(lX != nullptr);
stf_logical_data_set_symbol(lX, "X_composite");
stf_task_handle t = stf_task_create(ctx);
REQUIRE(t != nullptr);
stf_task_set_symbol(t, "T_composite");
stf_exec_place_handle e_place_dev0 = stf_exec_place_device(0);
stf_task_set_exec_place(t, e_place_dev0);
stf_exec_place_destroy(e_place_dev0);
stf_task_add_dep_with_dplace(t, lX, STF_RW, composite_dplace);
stf_task_start(t);
stf_task_end(t);
stf_task_destroy(t);
stf_data_place_destroy(composite_dplace);
stf_logical_data_destroy(lX);
stf_ctx_finalize(ctx);
for (size_t i = 0; i < N; ++i)
{
REQUIRE(X[i] == static_cast<float>(i));
}
}
C2H_TEST("composite data place with stf_exec_place_grid_create (vector of places + dim4)", "[task][places][composite]")
{
const size_t nplaces = 4;
stf_exec_place_handle places[4];
for (auto& place : places)
{
place = stf_exec_place_device(0);
}
stf_exec_place_handle grid_linear = stf_exec_place_grid_create(places, nplaces, nullptr);
REQUIRE(grid_linear != nullptr);
for (auto& place : places)
{
stf_exec_place_destroy(place);
}
stf_exec_place_grid_destroy(grid_linear);
for (auto& place : places)
{
place = stf_exec_place_device(0);
}
stf_dim4 grid_dims = {2, 2, 1, 1};
stf_exec_place_handle grid = stf_exec_place_grid_create(places, nplaces, &grid_dims);
REQUIRE(grid != nullptr);
for (auto& place : places)
{
stf_exec_place_destroy(place);
}
stf_data_place_handle composite_dplace = stf_data_place_composite(grid, blocked_mapper_1d);
REQUIRE(composite_dplace != nullptr);
stf_exec_place_grid_destroy(grid);
size_t N = 512;
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
std::vector<float> X(N);
for (size_t i = 0; i < N; ++i)
{
X[i] = static_cast<float>(i);
}
stf_logical_data_handle lX = stf_logical_data(ctx, X.data(), N * sizeof(float));
REQUIRE(lX != nullptr);
stf_task_handle t = stf_task_create(ctx);
REQUIRE(t != nullptr);
stf_exec_place_handle e_place = stf_exec_place_device(0);
stf_task_set_exec_place(t, e_place);
stf_exec_place_destroy(e_place);
stf_task_add_dep_with_dplace(t, lX, STF_RW, composite_dplace);
stf_task_start(t);
stf_task_end(t);
stf_task_destroy(t);
stf_data_place_destroy(composite_dplace);
stf_logical_data_destroy(lX);
stf_ctx_finalize(ctx);
for (size_t i = 0; i < N; ++i)
{
REQUIRE(X[i] == static_cast<float>(i));
}
}
C2H_TEST("task on exec_place_grid: get_grid_dims and get_custream_at_index", "[task][places][grid]")
{
const size_t nplaces = 2;
stf_exec_place_handle places[2];
for (auto& place : places)
{
place = stf_exec_place_device(0);
}
stf_exec_place_handle grid = stf_exec_place_grid_create(places, nplaces, nullptr);
REQUIRE(grid != nullptr);
for (auto& place : places)
{
stf_exec_place_destroy(place);
}
stf_data_place_handle composite_dplace = stf_data_place_composite(grid, blocked_mapper_1d);
REQUIRE(composite_dplace != nullptr);
stf_exec_place_set_affine_data_place(grid, composite_dplace);
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
std::vector<float> X(4, 0.0f);
stf_logical_data_handle lX = stf_logical_data(ctx, X.data(), X.size() * sizeof(float));
REQUIRE(lX != nullptr);
stf_task_handle t = stf_task_create(ctx);
REQUIRE(t != nullptr);
stf_task_set_exec_place(t, grid);
stf_task_add_dep(t, lX, STF_RW);
stf_task_start(t);
stf_dim4 dims;
int got_dims = stf_task_get_grid_dims(t, &dims);
REQUIRE(got_dims == 0);
REQUIRE(dims.x == 2);
REQUIRE(dims.y == 1);
REQUIRE(dims.z == 1);
REQUIRE(dims.t == 1);
CUstream s0, s1;
REQUIRE(stf_task_get_custream_at_index(t, 0, &s0) == 0);
REQUIRE(stf_task_get_custream_at_index(t, 1, &s1) == 0);
REQUIRE(s0 != nullptr);
REQUIRE(s1 != nullptr);
// Out-of-range linear index must report an error rather than reading past the stream grid.
CUstream s_oob;
REQUIRE(stf_task_get_custream_at_index(t, 2, &s_oob) != 0);
stf_task_end(t);
stf_task_destroy(t);
stf_data_place_destroy(composite_dplace);
stf_exec_place_grid_destroy(grid);
stf_logical_data_destroy(lX);
stf_ctx_finalize(ctx);
}
C2H_TEST("task get_grid_dims returns error for non-grid exec_place", "[task][places][grid]")
{
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
float val = 0.0f;
auto lX = stf_logical_data(ctx, &val, sizeof(float));
auto e_dev0 = stf_exec_place_device(0);
stf_task_handle t = stf_task_create(ctx);
REQUIRE(t != nullptr);
stf_task_set_exec_place(t, e_dev0);
stf_task_add_dep(t, lX, STF_RW);
stf_task_start(t);
stf_dim4 dims;
REQUIRE(stf_task_get_grid_dims(t, &dims) != 0);
stf_task_end(t);
stf_task_destroy(t);
stf_exec_place_destroy(e_dev0);
stf_logical_data_destroy(lX);
stf_ctx_finalize(ctx);
}
// ===== Place scope and accessor tests (task-free usage) =====
C2H_TEST("exec_place_scope enter/exit", "[places][scope]")
{
stf_machine_init();
stf_exec_place_handle dev0 = stf_exec_place_device(0);
REQUIRE(dev0 != nullptr);
stf_exec_place_scope_handle scope = stf_exec_place_scope_enter(dev0, 0);
REQUIRE(scope != nullptr);
stf_exec_place_scope_exit(scope);
stf_exec_place_scope_exit(nullptr);
stf_exec_place_destroy(dev0);
}
C2H_TEST("exec_place_scope nested", "[places][scope]")
{
stf_machine_init();
stf_exec_place_handle dev0 = stf_exec_place_device(0);
REQUIRE(dev0 != nullptr);
stf_exec_place_scope_handle outer = stf_exec_place_scope_enter(dev0, 0);
REQUIRE(outer != nullptr);
stf_exec_place_scope_handle inner = stf_exec_place_scope_enter(dev0, 0);
REQUIRE(inner != nullptr);
stf_exec_place_scope_exit(inner);
stf_exec_place_scope_exit(outer);
stf_exec_place_destroy(dev0);
}
C2H_TEST("exec_place_get_affine_data_place", "[places][accessor]")
{
stf_exec_place_handle dev0 = stf_exec_place_device(0);
REQUIRE(dev0 != nullptr);
stf_data_place_handle dp = stf_exec_place_get_affine_data_place(dev0);
REQUIRE(dp != nullptr);
REQUIRE(stf_data_place_get_device_ordinal(dp) == 0);
stf_data_place_destroy(dp);
stf_exec_place_destroy(dev0);
}
C2H_TEST("exec_place_pick_stream standalone", "[places][scope][stream]")
{
stf_machine_init();
// Standalone use: no STF context required, just a registry the caller owns.
stf_exec_place_resources_handle res = stf_exec_place_resources_create();
REQUIRE(res != nullptr);
stf_exec_place_handle dev0 = stf_exec_place_device(0);
REQUIRE(dev0 != nullptr);
stf_exec_place_scope_handle scope = stf_exec_place_scope_enter(dev0, 0);
REQUIRE(scope != nullptr);
CUstream s = stf_exec_place_pick_stream(res, dev0, /*for_computation=*/1);
REQUIRE(s != nullptr);
stf_exec_place_scope_exit(scope);
stf_exec_place_destroy(dev0);
stf_exec_place_resources_destroy(res);
}
C2H_TEST("exec_place resources are independent", "[places][scope][stream]")
{
stf_machine_init();
stf_exec_place_resources_handle res1 = stf_exec_place_resources_create();
stf_exec_place_resources_handle res2 = stf_exec_place_resources_create();
REQUIRE(res1 != nullptr);
REQUIRE(res2 != nullptr);
stf_exec_place_handle dev0 = stf_exec_place_device(0);
REQUIRE(dev0 != nullptr);
stf_exec_place_scope_handle scope = stf_exec_place_scope_enter(dev0, 0);
REQUIRE(scope != nullptr);
CUstream stream1 = stf_exec_place_pick_stream(res1, dev0, /*for_computation=*/1);
CUstream stream2 = stf_exec_place_pick_stream(res2, dev0, /*for_computation=*/1);
REQUIRE(stream1 != nullptr);
REQUIRE(stream2 != nullptr);
REQUIRE(stream1 != stream2);
stf_exec_place_scope_exit(scope);
stf_exec_place_destroy(dev0);
stf_exec_place_resources_destroy(res2);
stf_exec_place_resources_destroy(res1);
}
C2H_TEST("exec_place_pick_stream borrowed from context", "[places][scope][stream][ctx]")
{
stf_machine_init();
stf_ctx_handle ctx = stf_ctx_create();
stf_exec_place_resources_handle res = stf_ctx_get_place_resources(ctx);
REQUIRE(res != nullptr);
stf_exec_place_handle dev0 = stf_exec_place_device(0);
stf_exec_place_scope_handle scope = stf_exec_place_scope_enter(dev0, 0);
CUstream s = stf_exec_place_pick_stream(res, dev0, /*for_computation=*/1);
REQUIRE(s != nullptr);
stf_exec_place_scope_exit(scope);
stf_exec_place_destroy(dev0);
// `res` is a non-owning wrapper around context resources; destroy only the wrapper.
stf_exec_place_resources_destroy(res);
stf_ctx_finalize(ctx);
}
C2H_TEST("exec_place_get_place on grid", "[places][accessor][grid]")
{
const size_t nplaces = 2;
int device_ids[2] = {0, 0};
stf_exec_place_handle grid = stf_exec_place_grid_from_devices(device_ids, nplaces);
REQUIRE(grid != nullptr);
stf_exec_place_handle sub0 = stf_exec_place_get_place(grid, 0);
stf_exec_place_handle sub1 = stf_exec_place_get_place(grid, 1);
REQUIRE(sub0 != nullptr);
REQUIRE(sub1 != nullptr);
REQUIRE(stf_exec_place_is_device(sub0) != 0);
REQUIRE(stf_exec_place_is_device(sub1) != 0);
stf_exec_place_destroy(sub0);
stf_exec_place_destroy(sub1);
stf_exec_place_grid_destroy(grid);
}
C2H_TEST("exec_place_get_place on scalar", "[places][accessor]")
{
stf_exec_place_handle dev0 = stf_exec_place_device(0);
REQUIRE(dev0 != nullptr);
stf_exec_place_handle sub = stf_exec_place_get_place(dev0, 0);
REQUIRE(sub != nullptr);
REQUIRE(stf_exec_place_is_device(sub) != 0);
stf_exec_place_destroy(sub);
stf_exec_place_destroy(dev0);
}
C2H_TEST("exec_place_get_place out of bounds", "[places][accessor]")
{
stf_exec_place_handle dev0 = stf_exec_place_device(0);
REQUIRE(dev0 != nullptr);
REQUIRE(stf_exec_place_get_place(dev0, 1) == nullptr);
stf_exec_place_destroy(dev0);
int device_ids[2] = {0, 0};
stf_exec_place_handle grid = stf_exec_place_grid_from_devices(device_ids, 2);
REQUIRE(grid != nullptr);
REQUIRE(stf_exec_place_get_place(grid, 2) == nullptr);
stf_exec_place_grid_destroy(grid);
}
C2H_TEST("machine_init idempotent", "[places][machine]")
{
stf_machine_init();
stf_machine_init();
}
C2H_TEST("green_context_helper and green-context places", "[places][green_ctx]")
{
#if !defined(CUDART_VERSION) || CUDART_VERSION < 12040
REQUIRE(stf_green_context_helper_create(1, 0) == nullptr);
#else
stf_machine_init();
stf_green_context_helper_handle helper = stf_green_context_helper_create(1, 0);
if (helper == nullptr)
{
SKIP("green context support is not available");
}
REQUIRE(stf_green_context_helper_get_device_id(helper) == 0);
const size_t count = stf_green_context_helper_get_count(helper);
REQUIRE(count >= 1);
stf_exec_place_handle default_affine_ep = stf_exec_place_green_ctx(helper, 0, /*use_green_ctx_data_place=*/0);
REQUIRE(default_affine_ep != nullptr);
REQUIRE(stf_exec_place_is_device(default_affine_ep) != 0);
stf_data_place_handle default_affine_dp = stf_exec_place_get_affine_data_place(default_affine_ep);
REQUIRE(default_affine_dp != nullptr);
REQUIRE(stf_data_place_get_device_ordinal(default_affine_dp) == 0);
stf_exec_place_handle green_affine_ep = stf_exec_place_green_ctx(helper, 0, /*use_green_ctx_data_place=*/1);
REQUIRE(green_affine_ep != nullptr);
REQUIRE(stf_exec_place_is_device(green_affine_ep) != 0);
stf_data_place_handle green_affine_dp = stf_exec_place_get_affine_data_place(green_affine_ep);
REQUIRE(green_affine_dp != nullptr);
REQUIRE(stf_data_place_get_device_ordinal(green_affine_dp) == 0);
const std::string green_affine_desc = stf_data_place_to_string(green_affine_dp);
REQUIRE(green_affine_desc.find("green_ctx") != std::string::npos);
stf_data_place_handle green_dp = stf_data_place_green_ctx(helper, 0);
REQUIRE(green_dp != nullptr);
REQUIRE(stf_data_place_get_device_ordinal(green_dp) == 0);
REQUIRE(stf_data_place_allocation_is_stream_ordered(green_dp) == 1);
const std::string green_dp_desc = stf_data_place_to_string(green_dp);
REQUIRE(green_dp_desc.find("green_ctx") != std::string::npos);
REQUIRE(stf_exec_place_green_ctx(helper, count, /*use_green_ctx_data_place=*/0) == nullptr);
REQUIRE(stf_data_place_green_ctx(helper, count) == nullptr);
stf_data_place_destroy(green_dp);
stf_data_place_destroy(green_affine_dp);
stf_exec_place_destroy(green_affine_ep);
stf_data_place_destroy(default_affine_dp);
stf_exec_place_destroy(default_affine_ep);
stf_green_context_helper_destroy(helper);
#endif
}
C2H_TEST("data_place_allocate_device", "[places][allocate]")
{
stf_exec_place_resources_handle res = stf_exec_place_resources_create();
stf_exec_place_handle ep = stf_exec_place_device(0);
REQUIRE(ep != nullptr);
stf_exec_place_scope_handle scope = stf_exec_place_scope_enter(ep, 0);
REQUIRE(scope != nullptr);
CUstream stream = stf_exec_place_pick_stream(res, ep, /*for_computation=*/0);
stf_data_place_handle dplace = stf_exec_place_get_affine_data_place(ep);
REQUIRE(dplace != nullptr);
void* ptr = stf_data_place_allocate(dplace, 1024, reinterpret_cast<cudaStream_t>(stream));
REQUIRE(ptr != nullptr);
stf_data_place_deallocate(dplace, ptr, 1024, reinterpret_cast<cudaStream_t>(stream));
stf_data_place_destroy(dplace);
stf_exec_place_scope_exit(scope);
stf_exec_place_destroy(ep);
stf_exec_place_resources_destroy(res);
}
C2H_TEST("data_place_allocate_host", "[places][allocate]")
{
stf_data_place_handle dplace = stf_data_place_host();
REQUIRE(dplace != nullptr);
void* ptr = stf_data_place_allocate(dplace, 256, nullptr);
REQUIRE(ptr != nullptr);
int* buf = static_cast<int*>(ptr);
buf[0] = 42;
REQUIRE(buf[0] == 42);
stf_data_place_deallocate(dplace, ptr, 256, nullptr);
stf_data_place_destroy(dplace);
}
C2H_TEST("data_place_allocate_managed", "[places][allocate]")
{
stf_data_place_handle dplace = stf_data_place_managed();
REQUIRE(dplace != nullptr);
void* ptr = stf_data_place_allocate(dplace, 512, nullptr);
REQUIRE(ptr != nullptr);
int* buf = static_cast<int*>(ptr);
buf[0] = 99;
REQUIRE(buf[0] == 99);
stf_data_place_deallocate(dplace, ptr, 512, nullptr);
stf_data_place_destroy(dplace);
}
C2H_TEST("data_place_allocation_is_stream_ordered", "[places][allocate]")
{
stf_data_place_handle dev = stf_data_place_device(0);
REQUIRE(dev != nullptr);
REQUIRE(stf_data_place_allocation_is_stream_ordered(dev) == 1);
stf_data_place_destroy(dev);
stf_data_place_handle host = stf_data_place_host();
REQUIRE(host != nullptr);
REQUIRE(stf_data_place_allocation_is_stream_ordered(host) == 0);
stf_data_place_destroy(host);
stf_data_place_handle mgd = stf_data_place_managed();
REQUIRE(mgd != nullptr);
REQUIRE(stf_data_place_allocation_is_stream_ordered(mgd) == 0);
stf_data_place_destroy(mgd);
}
C2H_TEST("data_place_allocate_invalid_returns_null", "[places][allocate]")
{
stf_data_place_handle inv = stf_data_place_affine();
REQUIRE(inv != nullptr);
void* ptr = stf_data_place_allocate(inv, 64, nullptr);
REQUIRE(ptr == nullptr);
stf_data_place_destroy(inv);
}

761
cccl_upstream/c/experimental/stf/test/test_stackable.cu Normal file
View File

@@ -0,0 +1,761 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cmath>
#include <cstdint>
#include <cuda_runtime.h>
#include <c2h/catch2_test_helper.h>
#include <cccl/c/experimental/stf/stf.h>
__global__ void scale_kernel(int cnt, double* data, double factor)
{
const int tid = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
const int nthreads = static_cast<int>(gridDim.x * blockDim.x);
for (int i = tid; i < cnt; i += nthreads)
{
data[i] *= factor;
}
}
__global__ void increment_kernel(int cnt, double* data)
{
const int tid = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
const int nthreads = static_cast<int>(gridDim.x * blockDim.x);
for (int i = tid; i < cnt; i += nthreads)
{
data[i] += 1.0;
}
}
C2H_TEST("stackable: push_graph / pop", "[stackable]")
{
const size_t N = 256;
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
double* host_data;
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
for (size_t i = 0; i < N; i++)
{
host_data[i] = static_cast<double>(i);
}
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
REQUIRE(lA != nullptr);
// Multiply by 2 inside a nested graph scope.
stf_stackable_push_graph(ctx);
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
stf_task_enable_capture(t);
stf_task_start(t);
double* d = static_cast<double*>(stf_task_get(t, 0));
scale_kernel<<<2, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d, 2.0);
stf_task_end(t);
stf_task_destroy(t);
}
stf_stackable_pop(ctx);
stf_stackable_logical_data_destroy(lA);
stf_stackable_ctx_finalize(ctx);
for (size_t i = 0; i < N; i++)
{
REQUIRE(std::fabs(host_data[i] - 2.0 * static_cast<double>(i)) < 1e-10);
}
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
}
C2H_TEST("stackable: pop_prologue relaunch accumulates N times", "[stackable][launchable]")
{
const size_t N = 256;
const int relaunchN = 16;
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
double* host_data;
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
for (size_t i = 0; i < N; i++)
{
host_data[i] = 0.0;
}
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
REQUIRE(lA != nullptr);
stf_stackable_push_graph(ctx);
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
stf_task_enable_capture(t);
stf_task_start(t);
double* d = static_cast<double*>(stf_task_get(t, 0));
increment_kernel<<<2, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d);
stf_task_end(t);
stf_task_destroy(t);
}
// Two-phase pop: instantiate the graph, launch it relaunchN times, then
// run the epilogue to release resources and unfreeze lA.
stf_launchable_graph_handle lh = stf_stackable_pop_prologue(ctx);
REQUIRE(lh != nullptr);
for (int k = 0; k < relaunchN; ++k)
{
stf_launchable_graph_launch(lh);
}
stf_stackable_pop_epilogue(ctx);
stf_launchable_graph_destroy(lh);
stf_stackable_logical_data_destroy(lA);
stf_stackable_ctx_finalize(ctx);
for (size_t i = 0; i < N; i++)
{
REQUIRE(std::fabs(host_data[i] - static_cast<double>(relaunchN)) < 1e-10);
}
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
}
C2H_TEST("stackable: pop_prologue with zero launches unfreezes", "[stackable][launchable]")
{
const size_t N = 128;
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
double* host_data;
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
for (size_t i = 0; i < N; i++)
{
host_data[i] = 7.0;
}
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
REQUIRE(lA != nullptr);
// Push + submit work, but never launch the graph. The epilogue must still
// release resources so that lA is unfrozen and reusable below.
stf_stackable_push_graph(ctx);
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
stf_task_enable_capture(t);
stf_task_start(t);
double* d = static_cast<double*>(stf_task_get(t, 0));
increment_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d);
stf_task_end(t);
stf_task_destroy(t);
}
stf_launchable_graph_handle lh = stf_stackable_pop_prologue(ctx);
REQUIRE(lh != nullptr);
stf_stackable_pop_epilogue(ctx);
stf_launchable_graph_destroy(lh);
// Normal push_graph/pop still works after a zero-launch prologue+epilogue.
stf_stackable_push_graph(ctx);
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
stf_task_enable_capture(t);
stf_task_start(t);
double* d = static_cast<double*>(stf_task_get(t, 0));
scale_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d, 2.0);
stf_task_end(t);
stf_task_destroy(t);
}
stf_stackable_pop(ctx);
stf_stackable_logical_data_destroy(lA);
stf_stackable_ctx_finalize(ctx);
// Zero-launch means the first graph never ran. The second scope doubled
// the initial 7.0 to 14.0.
for (size_t i = 0; i < N; i++)
{
REQUIRE(std::fabs(host_data[i] - 14.0) < 1e-10);
}
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
}
C2H_TEST("stackable: launchable exec and stream accessors are non-null", "[stackable][launchable]")
{
const size_t N = 64;
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
double* host_data;
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
for (size_t i = 0; i < N; i++)
{
host_data[i] = 0.0;
}
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
REQUIRE(lA != nullptr);
stf_stackable_push_graph(ctx);
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
stf_task_enable_capture(t);
stf_task_start(t);
double* d = static_cast<double*>(stf_task_get(t, 0));
increment_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d);
stf_task_end(t);
stf_task_destroy(t);
}
stf_launchable_graph_handle lh = stf_stackable_pop_prologue(ctx);
REQUIRE(lh != nullptr);
// Accessors must be valid between prologue and epilogue. graph() must
// return a live cudaGraph_t without forcing instantiation, exec() must
// return a live cudaGraphExec_t, stream() is pure observation.
REQUIRE(stf_launchable_graph_graph(lh) != nullptr);
REQUIRE(stf_launchable_graph_exec(lh) != nullptr);
REQUIRE(stf_launchable_graph_stream(lh) != nullptr);
stf_launchable_graph_launch(lh);
stf_stackable_pop_epilogue(ctx);
stf_launchable_graph_destroy(lh);
stf_stackable_logical_data_destroy(lA);
stf_stackable_ctx_finalize(ctx);
for (size_t i = 0; i < N; i++)
{
REQUIRE(std::fabs(host_data[i] - 1.0) < 1e-10);
}
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
}
C2H_TEST("stackable: launchable graph() embed into outer graph", "[stackable][launchable]")
{
const size_t N = 64;
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
double* host_data;
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
for (size_t i = 0; i < N; i++)
{
host_data[i] = 0.0;
}
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
REQUIRE(lA != nullptr);
stf_stackable_push_graph(ctx);
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
stf_task_enable_capture(t);
stf_task_start(t);
double* d = static_cast<double*>(stf_task_get(t, 0));
increment_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d);
stf_task_end(t);
stf_task_destroy(t);
}
stf_launchable_graph_handle lh = stf_stackable_pop_prologue(ctx);
REQUIRE(lh != nullptr);
// Grab the underlying cudaGraph_t WITHOUT forcing instantiation and
// without ever calling stf_launchable_graph_exec(). The child graph
// built by the nested scope is embedded into an outer graph which is
// instantiated and launched manually here.
cudaGraph_t child_graph = stf_launchable_graph_graph(lh);
REQUIRE(child_graph != nullptr);
cudaStream_t support_stream = stf_launchable_graph_stream(lh);
REQUIRE(support_stream != nullptr);
cudaGraph_t outer = nullptr;
REQUIRE(cudaGraphCreate(&outer, 0) == cudaSuccess);
cudaGraphNode_t child_node = nullptr;
REQUIRE(cudaGraphAddChildGraphNode(&child_node, outer, nullptr, 0, child_graph) == cudaSuccess);
cudaGraphExec_t outer_exec = nullptr;
#if _CCCL_CTK_AT_LEAST(12, 0)
REQUIRE(cudaGraphInstantiate(&outer_exec, outer, 0) == cudaSuccess);
#else
REQUIRE(cudaGraphInstantiate(&outer_exec, outer, nullptr, nullptr, 0) == cudaSuccess);
#endif
// Route the outer launch through the support stream: since graph() has
// triggered the lazy dep-A sync on that stream, it is safe to drive
// cudaGraphLaunch on it here.
REQUIRE(cudaGraphLaunch(outer_exec, support_stream) == cudaSuccess);
REQUIRE(cudaGraphExecDestroy(outer_exec) == cudaSuccess);
REQUIRE(cudaGraphDestroy(outer) == cudaSuccess);
stf_stackable_pop_epilogue(ctx);
stf_launchable_graph_destroy(lh);
stf_stackable_logical_data_destroy(lA);
stf_stackable_ctx_finalize(ctx);
for (size_t i = 0; i < N; i++)
{
REQUIRE(std::fabs(host_data[i] - 1.0) < 1e-10);
}
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
}
C2H_TEST("stackable: shared pop_prologue dup/free releases only at last free", "[stackable][launchable]")
{
const size_t N = 128;
const int relaunchN = 5;
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
double* host_data;
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
for (size_t i = 0; i < N; i++)
{
host_data[i] = 0.0;
}
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
REQUIRE(lA != nullptr);
stf_stackable_push_graph(ctx);
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
stf_task_enable_capture(t);
stf_task_start(t);
double* d = static_cast<double*>(stf_task_get(t, 0));
increment_kernel<<<2, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d);
stf_task_end(t);
stf_task_destroy(t);
}
stf_launchable_graph_shared h1 = nullptr;
REQUIRE(stf_stackable_pop_prologue_shared(ctx, &h1) == 0);
REQUIRE(h1 != nullptr);
REQUIRE(stf_launchable_graph_shared_valid(h1) == 1);
REQUIRE(stf_launchable_graph_shared_stream(h1) != nullptr);
// Dup before launching anything: both handles must be able to drive the
// same underlying graph.
stf_launchable_graph_shared h2 = nullptr;
REQUIRE(stf_launchable_graph_shared_dup(h1, &h2) == 0);
REQUIRE(h2 != nullptr);
REQUIRE(stf_launchable_graph_shared_valid(h2) == 1);
for (int k = 0; k < relaunchN; ++k)
{
// Alternate between the two handles - both must work.
if ((k & 1) == 0)
{
stf_launchable_graph_shared_launch(h1);
}
else
{
stf_launchable_graph_shared_launch(h2);
}
}
// Free one handle; the other must still launch. No pop_epilogue yet.
stf_launchable_graph_shared_free(h1);
REQUIRE(stf_launchable_graph_shared_valid(h2) == 1);
stf_launchable_graph_shared_launch(h2);
// Free the last handle: pop_epilogue runs automatically here.
stf_launchable_graph_shared_free(h2);
// The context must be usable again after the shared release.
stf_stackable_push_graph(ctx);
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
stf_task_enable_capture(t);
stf_task_start(t);
double* d = static_cast<double*>(stf_task_get(t, 0));
scale_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d, 2.0);
stf_task_end(t);
stf_task_destroy(t);
}
stf_stackable_pop(ctx);
stf_stackable_logical_data_destroy(lA);
stf_stackable_ctx_finalize(ctx);
// Each launch added +1; final scale doubled; relaunchN launches via h1/h2
// plus one extra launch via h2 after free(h1) -> (relaunchN + 1) * 2.
const double expected = 2.0 * (static_cast<double>(relaunchN) + 1.0);
for (size_t i = 0; i < N; i++)
{
REQUIRE(std::fabs(host_data[i] - expected) < 1e-10);
}
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
}
C2H_TEST("stackable: shared pop_prologue tolerates NULL free", "[stackable][launchable]")
{
// stf_launchable_graph_shared_free(NULL) must be a no-op just like the
// other destroy entry points. The valid() probe returns 0 for NULL.
stf_launchable_graph_shared_free(nullptr);
REQUIRE(stf_launchable_graph_shared_valid(nullptr) == 0);
}
C2H_TEST("stackable: nested push_graph scopes", "[stackable]")
{
const size_t N = 128;
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
double* host_data;
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
for (size_t i = 0; i < N; i++)
{
host_data[i] = 0.0;
}
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
REQUIRE(lA != nullptr);
// Two nested scopes: each scope adds 1.0, so after popping both we expect 2.0.
stf_stackable_push_graph(ctx);
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
stf_task_enable_capture(t);
stf_task_start(t);
double* d = static_cast<double*>(stf_task_get(t, 0));
increment_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(static_cast<int>(N), d);
stf_task_end(t);
stf_task_destroy(t);
stf_stackable_push_graph(ctx);
{
stf_task_handle t2 = stf_stackable_task_create(ctx);
REQUIRE(t2 != nullptr);
stf_stackable_task_add_dep(ctx, t2, lA, STF_RW);
stf_task_enable_capture(t2);
stf_task_start(t2);
double* d2 = static_cast<double*>(stf_task_get(t2, 0));
increment_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t2)>>>(static_cast<int>(N), d2);
stf_task_end(t2);
stf_task_destroy(t2);
}
stf_stackable_pop(ctx);
}
stf_stackable_pop(ctx);
stf_stackable_logical_data_destroy(lA);
stf_stackable_ctx_finalize(ctx);
for (size_t i = 0; i < N; i++)
{
REQUIRE(std::fabs(host_data[i] - 2.0) < 1e-10);
}
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
}
C2H_TEST("stackable: token + fence", "[stackable]")
{
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
stf_logical_data_handle tok = stf_stackable_token(ctx);
REQUIRE(tok != nullptr);
// Sequential task chain through the token: t1 (write) -> t2 (read).
stf_task_handle t1 = stf_stackable_task_create(ctx);
REQUIRE(t1 != nullptr);
stf_stackable_task_add_dep(ctx, t1, tok, STF_WRITE);
stf_task_start(t1);
stf_task_end(t1);
stf_task_destroy(t1);
stf_task_handle t2 = stf_stackable_task_create(ctx);
REQUIRE(t2 != nullptr);
stf_stackable_task_add_dep(ctx, t2, tok, STF_READ);
stf_task_start(t2);
stf_task_end(t2);
stf_task_destroy(t2);
cudaStream_t fence = stf_stackable_ctx_fence(ctx);
REQUIRE(cudaStreamSynchronize(fence) == cudaSuccess);
stf_stackable_token_destroy(tok);
stf_stackable_ctx_finalize(ctx);
}
#if _CCCL_CTK_AT_LEAST(12, 4)
// Smoke test for the while/repeat C-API surface: create+destroy each kind of
// scope without populating a body. Body-level integration is exercised at the
// C++ level by cudax/test/stf/local_stf/stackable_nested_repeat.cu and the
// graph_scope_test, but the C-API task-driven body still needs a follow-up to
// nail down the right capture path; tracked separately.
C2H_TEST("stackable: push_repeat / pop_repeat smoke", "[stackable][repeat]")
{
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
stf_repeat_scope_handle scope = stf_stackable_push_repeat(ctx, /*count=*/1);
REQUIRE(scope != nullptr);
stf_stackable_pop_repeat(scope);
stf_stackable_ctx_finalize(ctx);
}
C2H_TEST("stackable: push_while / pop_while smoke", "[stackable][while]")
{
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
stf_while_scope_handle scope = stf_stackable_push_while(ctx);
REQUIRE(scope != nullptr);
// The conditional handle is observable as a uint64_t; just sanity-check it.
REQUIRE(stf_while_scope_get_cond_handle(scope) != 0);
stf_stackable_pop_while(scope);
stf_stackable_ctx_finalize(ctx);
}
// Regression test mirroring probe_k_sweep.py: inside a while-scope body, chain
// K tasks that each do .rw() on the same persistent logical data, and make the
// loop execute exactly once. Sweep K=1..16 and expect every element of the
// accumulator to equal K. The equivalent Python probe fails deterministically
// when K is a multiple of 4 (drops exactly one update), so this test pins down
// whether the bug is in the C-API task path or somewhere above it.
C2H_TEST("stackable: while-body K chained rw tasks sweep", "[stackable][while][c-api]")
{
const int Nd = 128;
const double tol_eps = 1e-10;
int total_mismatches = 0;
double total_off_by_one = 0.0;
for (int K = 1; K <= 16; ++K)
{
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
// Accumulator: zero-initialized double[Nd].
double* host_acc;
REQUIRE(cudaMallocHost(&host_acc, Nd * sizeof(double)) == cudaSuccess);
for (int i = 0; i < Nd; i++)
{
host_acc[i] = 0.0;
}
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_acc, Nd * sizeof(double));
REQUIRE(lA != nullptr);
// "done" flag: starts at 1.0, body drives it to 0.0 so while stops after 1
// iteration. We use a double scalar to keep it consistent with the kernel
// family used by the probe.
double* host_done;
REQUIRE(cudaMallocHost(&host_done, sizeof(double)) == cudaSuccess);
host_done[0] = 1.0;
stf_logical_data_handle lD = stf_stackable_logical_data(ctx, host_done, sizeof(double));
REQUIRE(lD != nullptr);
stf_while_scope_handle scope = stf_stackable_push_while(ctx);
REQUIRE(scope != nullptr);
{
// K chained increments on lA, using the C-API raw task path that the
// Python binding also uses.
for (int k = 0; k < K; ++k)
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
stf_task_enable_capture(t);
stf_task_start(t);
double* d = static_cast<double*>(stf_task_get(t, 0));
increment_kernel<<<1, 64, 0, (cudaStream_t) stf_task_get_custream(t)>>>(Nd, d);
stf_task_end(t);
stf_task_destroy(t);
}
// Drive the done flag to 0.0 so the loop stops after 1 iteration.
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, lD, STF_WRITE);
stf_task_enable_capture(t);
stf_task_start(t);
double* d = static_cast<double*>(stf_task_get(t, 0));
scale_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>(1, d, 0.0);
stf_task_end(t);
stf_task_destroy(t);
}
// Continue while done > 0.5 (i.e. stop after we've zeroed it).
stf_stackable_while_cond_scalar(ctx, scope, lD, STF_CMP_GT, 0.5, STF_DTYPE_FLOAT64);
}
stf_stackable_pop_while(scope);
stf_stackable_logical_data_destroy(lA);
stf_stackable_logical_data_destroy(lD);
stf_stackable_ctx_finalize(ctx);
const double expected = static_cast<double>(K);
int mismatches = 0;
for (int i = 0; i < Nd; i++)
{
if (std::fabs(host_acc[i] - expected) > tol_eps)
{
++mismatches;
}
}
if (mismatches != 0)
{
fprintf(stderr,
"[C-API K=%d] host_acc[0]=%g expected=%g (%d/%d mismatches)\n",
K,
host_acc[0],
expected,
mismatches,
Nd);
total_mismatches += mismatches;
total_off_by_one += host_acc[0] - expected;
}
REQUIRE(cudaFreeHost(host_acc) == cudaSuccess);
REQUIRE(cudaFreeHost(host_done) == cudaSuccess);
}
REQUIRE(total_mismatches == 0);
(void) total_off_by_one;
}
namespace
{
// Run a while loop whose body increments a 1-element double counter once per
// iteration and leaves a 1-element flag at its initial value 1.0. The
// continuation condition is built by `set_condition` from the counter and
// flag handles. Returns the final counter value observed on the host.
template <typename SetCondition>
double run_compound_while(SetCondition&& set_condition)
{
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
double* host_iter;
REQUIRE(cudaMallocHost(&host_iter, sizeof(double)) == cudaSuccess);
host_iter[0] = 0.0;
double* host_flag;
REQUIRE(cudaMallocHost(&host_flag, sizeof(double)) == cudaSuccess);
host_flag[0] = 1.0;
stf_logical_data_handle lIter = stf_stackable_logical_data(ctx, host_iter, sizeof(double));
REQUIRE(lIter != nullptr);
stf_logical_data_handle lFlag = stf_stackable_logical_data(ctx, host_flag, sizeof(double));
REQUIRE(lFlag != nullptr);
stf_while_scope_handle scope = stf_stackable_push_while(ctx);
REQUIRE(scope != nullptr);
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, lIter, STF_RW);
stf_task_enable_capture(t);
stf_task_start(t);
double* d = static_cast<double*>(stf_task_get(t, 0));
increment_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>(1, d);
stf_task_end(t);
stf_task_destroy(t);
set_condition(ctx, scope, lIter, lFlag);
}
stf_stackable_pop_while(scope);
stf_stackable_logical_data_destroy(lIter);
stf_stackable_logical_data_destroy(lFlag);
stf_stackable_ctx_finalize(ctx);
const double result = host_iter[0];
REQUIRE(cudaFreeHost(host_iter) == cudaSuccess);
REQUIRE(cudaFreeHost(host_flag) == cudaSuccess);
return result;
}
} // namespace
C2H_TEST("stackable: while compound condition", "[stackable][while][c-api]")
{
SECTION("ALL combiner stops at the iteration cap")
{
// flag > 0.5 is always true; iter < 5 caps the loop at 5 iterations.
const double iters = run_compound_while(
[](stf_ctx_handle ctx, stf_while_scope_handle scope, stf_logical_data_handle lIter, stf_logical_data_handle lFlag) {
stf_while_cond_term terms[2] = {
{lFlag, STF_CMP_GT, 0.5, STF_DTYPE_FLOAT64, 0},
{lIter, STF_CMP_LT, 5.0, STF_DTYPE_FLOAT64, 0},
};
stf_stackable_while_cond_multi(ctx, scope, terms, 2, STF_COND_ALL);
});
REQUIRE(iters == 5.0);
}
SECTION("ANY combiner with a negated term")
{
// ~(flag > 0.5) is always false, so only iter < 3 keeps the loop going.
const double iters = run_compound_while(
[](stf_ctx_handle ctx, stf_while_scope_handle scope, stf_logical_data_handle lIter, stf_logical_data_handle lFlag) {
stf_while_cond_term terms[2] = {
{lIter, STF_CMP_LT, 3.0, STF_DTYPE_FLOAT64, 0},
{lFlag, STF_CMP_GT, 0.5, STF_DTYPE_FLOAT64, 1},
};
stf_stackable_while_cond_multi(ctx, scope, terms, 2, STF_COND_ANY);
});
REQUIRE(iters == 3.0);
}
SECTION("duplicate logical data across terms shares one dependency")
{
const double iters = run_compound_while(
[](stf_ctx_handle ctx,
stf_while_scope_handle scope,
stf_logical_data_handle lIter,
stf_logical_data_handle /*lFlag*/) {
stf_while_cond_term terms[2] = {
{lIter, STF_CMP_LT, 4.0, STF_DTYPE_FLOAT64, 0},
{lIter, STF_CMP_GT, -1.0, STF_DTYPE_FLOAT64, 0},
};
stf_stackable_while_cond_multi(ctx, scope, terms, 2, STF_COND_ALL);
});
REQUIRE(iters == 4.0);
}
}
#endif // _CCCL_CTK_AT_LEAST(12, 4)

237
cccl_upstream/c/experimental/stf/test/test_stackable_token_push.cu Normal file
View File

@@ -0,0 +1,237 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
//
// Regression test for the C-facade stackable-token dispatch fix. Combining a
// stackable token with push_graph / pop_prologue used to abort inside STF with
// a "Data interface type mismatch" (assumed void_interface, actual
// mdspan<char, ..., layout_stride>) because the C API treated every stackable
// logical-data handle as a slice<char> and mis-cast tokens. The abort was a
// hard C-level abort, so the Python binding that drives this exact sequence
// could not catch it:
//
// ctx = stf.stackable_context()
// tok = ctx.token()
// ctx.push()
// with ctx.task(tok.write()): ...
// with ctx.task(tok.read()): ...
// step_graph = ctx.pop_prologue_shared()
//
// These tests drive the same sequences through the C stackable API directly,
// so the path stays covered without any Python / Warp in the picture.
//
// The existing `stackable: token + fence` test uses tokens but outside any
// push_graph scope, and the existing pop_prologue tests use real logical_data;
// so the combination "tokens inside push_graph" is only exercised here.
#include <cmath>
#include <cstdint>
#include <cuda_runtime.h>
#include <c2h/catch2_test_helper.h>
#include <cccl/c/experimental/stf/stf.h>
namespace
{
__global__ void noop_kernel() {}
} // namespace
// Minimal case: a single token-only task inside a push_graph / pop scope.
// Does NOT use pop_prologue — just push_graph + pop — so token task-deps
// handling is covered independently of the prologue machinery.
C2H_TEST("stackable: token in push_graph scope (no prologue)", "[stackable][token][bug]")
{
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
stf_logical_data_handle tok = stf_stackable_token(ctx);
REQUIRE(tok != nullptr);
stf_stackable_push_graph(ctx);
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, tok, STF_WRITE);
stf_task_enable_capture(t);
stf_task_start(t);
noop_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>();
stf_task_end(t);
stf_task_destroy(t);
}
stf_stackable_pop(ctx);
stf_stackable_token_destroy(tok);
stf_stackable_ctx_finalize(ctx);
}
// Exact mirror of the Python run_stf_unified path:
// ctx.push() -> task(tok.write()) -> task(tok.read()) -> pop_prologue(_shared)
// This used to abort inside pop_prologue(_shared)() with the void_interface vs
// mdspan<char> mismatch before the C-facade dispatch fix.
C2H_TEST("stackable: token write/read chain + pop_prologue", "[stackable][token][launchable][bug]")
{
const int relaunchN = 4;
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
stf_logical_data_handle tok = stf_stackable_token(ctx);
REQUIRE(tok != nullptr);
stf_stackable_push_graph(ctx);
{
// Writer task (equivalent of Python `tok.write()`).
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, tok, STF_WRITE);
stf_task_enable_capture(t);
stf_task_start(t);
noop_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>();
stf_task_end(t);
stf_task_destroy(t);
}
// Reader task (equivalent of Python `tok.read()`).
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, tok, STF_READ);
stf_task_enable_capture(t);
stf_task_start(t);
noop_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>();
stf_task_end(t);
stf_task_destroy(t);
}
}
stf_launchable_graph_handle lh = stf_stackable_pop_prologue(ctx);
REQUIRE(lh != nullptr);
for (int k = 0; k < relaunchN; ++k)
{
stf_launchable_graph_launch(lh);
}
stf_stackable_pop_epilogue(ctx);
stf_launchable_graph_destroy(lh);
stf_stackable_token_destroy(tok);
stf_stackable_ctx_finalize(ctx);
}
// Same as above but using the shared flavour of pop_prologue, which is what
// the Python `pop_prologue_shared()` binding calls into.
C2H_TEST("stackable: token write/read chain + pop_prologue_shared", "[stackable][token][launchable][bug]")
{
const int relaunchN = 4;
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
stf_logical_data_handle tok = stf_stackable_token(ctx);
REQUIRE(tok != nullptr);
stf_stackable_push_graph(ctx);
{
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, tok, STF_WRITE);
stf_task_enable_capture(t);
stf_task_start(t);
noop_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>();
stf_task_end(t);
stf_task_destroy(t);
}
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, tok, STF_READ);
stf_task_enable_capture(t);
stf_task_start(t);
noop_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>();
stf_task_end(t);
stf_task_destroy(t);
}
}
stf_launchable_graph_shared h = nullptr;
REQUIRE(stf_stackable_pop_prologue_shared(ctx, &h) == 0);
REQUIRE(h != nullptr);
for (int k = 0; k < relaunchN; ++k)
{
stf_launchable_graph_shared_launch(h);
}
// Last free drops the strong ref and runs pop_epilogue automatically.
stf_launchable_graph_shared_free(h);
stf_stackable_token_destroy(tok);
stf_stackable_ctx_finalize(ctx);
}
// Sanity check: replacing the token with a real logical_data in the same
// push_graph + pop_prologue shape *should* work. This matches the
// `run_stf_unified_ld` workaround that the Python mockup confirmed OK.
C2H_TEST("stackable: logical_data write/read chain + pop_prologue (workaround)", "[stackable][launchable]")
{
const size_t N = 8;
const int relaunchN = 4;
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
uint8_t* host_dep = nullptr;
cudaError_t err = cudaMallocHost(&host_dep, N * sizeof(uint8_t));
REQUIRE(err == cudaSuccess);
for (size_t i = 0; i < N; i++)
{
host_dep[i] = 0;
}
stf_logical_data_handle ld = stf_stackable_logical_data(ctx, host_dep, N * sizeof(uint8_t));
REQUIRE(ld != nullptr);
stf_stackable_push_graph(ctx);
{
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, ld, STF_RW);
stf_task_enable_capture(t);
stf_task_start(t);
noop_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>();
stf_task_end(t);
stf_task_destroy(t);
}
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, ld, STF_READ);
stf_task_enable_capture(t);
stf_task_start(t);
noop_kernel<<<1, 1, 0, (cudaStream_t) stf_task_get_custream(t)>>>();
stf_task_end(t);
stf_task_destroy(t);
}
}
stf_launchable_graph_handle lh = stf_stackable_pop_prologue(ctx);
REQUIRE(lh != nullptr);
for (int k = 0; k < relaunchN; ++k)
{
stf_launchable_graph_launch(lh);
}
stf_stackable_pop_epilogue(ctx);
stf_launchable_graph_destroy(lh);
stf_stackable_logical_data_destroy(ld);
stf_stackable_ctx_finalize(ctx);
REQUIRE(cudaFreeHost(host_dep) == cudaSuccess);
}

314
cccl_upstream/c/experimental/stf/test/test_stream_ctx_override.cu Normal file
View File

@@ -0,0 +1,314 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
// Minimal tests for stf_ctx_create_ex() with has_stream=1 (caller-provided
// CUDA stream) on the stream backend, with no async_resources handle shared
// across contexts. These verify that contexts created back-to-back on the same
// caller stream chain their work transitively through that stream.
#include <vector>
#include <cuda_runtime.h>
#include <c2h/catch2_test_helper.h>
#include <cccl/c/experimental/stf/stf.h>
namespace
{
// A device sink that is written but never read. Publishing the busy-loop
// result here gives the loop an observable side effect, so the compiler
// cannot optimize it away, without perturbing the result buffer.
__device__ unsigned g_busy_sink;
// Writes `value` into every slot of `arr`. The inner busy loop widens the
// kernel window so that a failure to chain ctx2-after-ctx1 is observable:
// ctx1 is still running when ctx2's kernel races in.
__global__ void slow_set_kernel(int* arr, int n, int value, int iters)
{
const int tid = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
if (tid >= n)
{
return;
}
// Busy loop to keep the kernel resident on the SM for a while. `acc` is
// unsigned so the accumulation wraps with well-defined behavior.
unsigned acc = 0;
for (int i = 0; i < iters; ++i)
{
acc += (static_cast<unsigned>(i) * 1103515245u + 12345u) & 0x7fffffffu;
}
// Publish `acc` via an atomic: an observable, race-free side effect that
// keeps the loop alive while the stored result stays exactly `value`.
atomicAdd(&g_busy_sink, acc);
arr[tid] = value;
}
void submit_set(stf_ctx_handle ctx, int* d_arr, int n, int value, int iters)
{
stf_logical_data_handle tok = stf_token(ctx);
REQUIRE(tok != nullptr);
stf_logical_data_set_symbol(tok, "tok");
stf_task_handle t = stf_task_create(ctx);
REQUIRE(t != nullptr);
stf_task_set_symbol(t, "slow_set");
stf_task_add_dep(t, tok, STF_RW);
stf_task_start(t);
CUstream s = stf_task_get_custream(t);
REQUIRE(s != nullptr);
const int threads = 128;
const int blocks = (n + threads - 1) / threads;
slow_set_kernel<<<blocks, threads, 0, (cudaStream_t) s>>>(d_arr, n, value, iters);
stf_task_end(t);
stf_task_destroy(t);
stf_logical_data_destroy(tok);
}
// Submits `K` concurrent token-tasks in a single context; each writes `value`
// into its own slice of `d_arr`. Multiple independent tokens per context make
// STF spread kernels across several pool streams, so ordering depends on the
// caller-stream chaining contract.
void run_ctx_k_concurrent(cudaStream_t s, int* d_arr, int N, int K, int value, int iters)
{
stf_ctx_options opts{};
opts.backend = STF_BACKEND_STREAM;
opts.has_stream = 1;
opts.stream = s;
opts.handle = nullptr;
stf_ctx_handle ctx = stf_ctx_create_ex(&opts);
REQUIRE(ctx != nullptr);
const int per = N / K;
for (int k = 0; k < K; ++k)
{
stf_logical_data_handle tok = stf_token(ctx);
REQUIRE(tok != nullptr);
stf_task_handle t = stf_task_create(ctx);
REQUIRE(t != nullptr);
stf_task_add_dep(t, tok, STF_RW);
stf_task_start(t);
CUstream ts = stf_task_get_custream(t);
const int threads = 128;
const int blocks = (per + threads - 1) / threads;
int* slice = d_arr + k * per;
slow_set_kernel<<<blocks, threads, 0, (cudaStream_t) ts>>>(slice, per, value, iters);
stf_task_end(t);
stf_task_destroy(t);
stf_logical_data_destroy(tok);
}
stf_ctx_finalize(ctx);
}
// More faithful MLP mimic: K concurrent tokens, each with T chained tasks
// (sequential RW on the same token), so each token effectively owns a chain of
// T slow kernels on one pool stream.
void run_ctx_k_chains(cudaStream_t s, int* d_arr, int N, int K, int chain_len, int value, int iters)
{
stf_ctx_options opts{};
opts.backend = STF_BACKEND_STREAM;
opts.has_stream = 1;
opts.stream = s;
opts.handle = nullptr;
stf_ctx_handle ctx = stf_ctx_create_ex(&opts);
REQUIRE(ctx != nullptr);
const int per = N / K;
std::vector<stf_logical_data_handle> toks(K);
for (int k = 0; k < K; ++k)
{
toks[k] = stf_token(ctx);
REQUIRE(toks[k] != nullptr);
}
for (int step = 0; step < chain_len; ++step)
{
for (int k = 0; k < K; ++k)
{
stf_task_handle t = stf_task_create(ctx);
REQUIRE(t != nullptr);
stf_task_add_dep(t, toks[k], STF_RW);
stf_task_start(t);
CUstream ts = stf_task_get_custream(t);
const int threads = 128;
const int blocks = (per + threads - 1) / threads;
int* slice = d_arr + k * per;
slow_set_kernel<<<blocks, threads, 0, (cudaStream_t) ts>>>(slice, per, value, iters);
stf_task_end(t);
stf_task_destroy(t);
}
}
for (int k = 0; k < K; ++k)
{
stf_logical_data_destroy(toks[k]);
}
stf_ctx_finalize(ctx);
}
} // namespace
C2H_TEST("stf_ctx_create_ex: 1 token per context, back-to-back, stream-only", "[context][stream]")
{
constexpr int N = 1 << 14;
constexpr int ITERS = 1 << 18;
cudaStream_t s{};
REQUIRE(cudaStreamCreate(&s) == cudaSuccess);
int* d_arr = nullptr;
REQUIRE(cudaMalloc(&d_arr, N * sizeof(int)) == cudaSuccess);
REQUIRE(cudaMemsetAsync(d_arr, 0, N * sizeof(int), s) == cudaSuccess);
for (int iter = 0; iter < 20; ++iter)
{
{
stf_ctx_options opts{};
opts.backend = STF_BACKEND_STREAM;
opts.has_stream = 1;
opts.stream = s;
opts.handle = nullptr;
stf_ctx_handle ctx = stf_ctx_create_ex(&opts);
REQUIRE(ctx != nullptr);
submit_set(ctx, d_arr, N, /*value=*/1, ITERS);
stf_ctx_finalize(ctx);
}
{
stf_ctx_options opts{};
opts.backend = STF_BACKEND_STREAM;
opts.has_stream = 1;
opts.stream = s;
opts.handle = nullptr;
stf_ctx_handle ctx = stf_ctx_create_ex(&opts);
REQUIRE(ctx != nullptr);
submit_set(ctx, d_arr, N, /*value=*/2, ITERS);
stf_ctx_finalize(ctx);
}
REQUIRE(cudaStreamSynchronize(s) == cudaSuccess);
int h_arr[16]{};
REQUIRE(cudaMemcpy(h_arr, d_arr, sizeof(h_arr), cudaMemcpyDeviceToHost) == cudaSuccess);
for (int i = 0; i < static_cast<int>(sizeof(h_arr) / sizeof(int)); ++i)
{
INFO("iter=" << iter << " i=" << i << " value=" << h_arr[i]);
REQUIRE(h_arr[i] == 2);
}
}
REQUIRE(cudaFree(d_arr) == cudaSuccess);
REQUIRE(cudaStreamDestroy(s) == cudaSuccess);
}
C2H_TEST("stf_ctx_create_ex: K chains of T tasks per token, back-to-back, stream-only, no handle",
"[context][stream][tokens][lifetime]")
{
constexpr int N = 1 << 16;
constexpr int K = 8;
constexpr int CHAIN_LEN = 20;
constexpr int ITERS = 1 << 18;
cudaStream_t s{};
REQUIRE(cudaStreamCreate(&s) == cudaSuccess);
int* d_arr = nullptr;
REQUIRE(cudaMalloc(&d_arr, N * sizeof(int)) == cudaSuccess);
REQUIRE(cudaMemsetAsync(d_arr, 0, N * sizeof(int), s) == cudaSuccess);
for (int iter = 0; iter < 20; ++iter)
{
run_ctx_k_chains(s, d_arr, N, K, CHAIN_LEN, /*value=*/1, ITERS);
run_ctx_k_chains(s, d_arr, N, K, CHAIN_LEN, /*value=*/2, ITERS);
REQUIRE(cudaStreamSynchronize(s) == cudaSuccess);
std::vector<int> h_arr(N, 0);
REQUIRE(cudaMemcpy(h_arr.data(), d_arr, N * sizeof(int), cudaMemcpyDeviceToHost) == cudaSuccess);
int mismatches = 0;
int first_bad_i = -1;
int first_bad_v = 0;
for (int i = 0; i < N; ++i)
{
if (h_arr[i] != 2)
{
++mismatches;
if (first_bad_i < 0)
{
first_bad_i = i;
first_bad_v = h_arr[i];
}
}
}
INFO("iter=" << iter << " mismatches=" << mismatches << " first_bad_idx=" << first_bad_i
<< " first_bad_val=" << first_bad_v);
REQUIRE(mismatches == 0);
}
REQUIRE(cudaFree(d_arr) == cudaSuccess);
REQUIRE(cudaStreamDestroy(s) == cudaSuccess);
}
C2H_TEST("stf_ctx_create_ex: K concurrent tokens per context, back-to-back, stream-only", "[context][stream][tokens]")
{
constexpr int N = 1 << 16;
constexpr int K = 8;
constexpr int ITERS = 1 << 18;
cudaStream_t s{};
REQUIRE(cudaStreamCreate(&s) == cudaSuccess);
int* d_arr = nullptr;
REQUIRE(cudaMalloc(&d_arr, N * sizeof(int)) == cudaSuccess);
REQUIRE(cudaMemsetAsync(d_arr, 0, N * sizeof(int), s) == cudaSuccess);
for (int iter = 0; iter < 20; ++iter)
{
run_ctx_k_concurrent(s, d_arr, N, K, /*value=*/1, ITERS);
run_ctx_k_concurrent(s, d_arr, N, K, /*value=*/2, ITERS);
REQUIRE(cudaStreamSynchronize(s) == cudaSuccess);
std::vector<int> h_arr(N, 0);
REQUIRE(cudaMemcpy(h_arr.data(), d_arr, N * sizeof(int), cudaMemcpyDeviceToHost) == cudaSuccess);
int mismatches = 0;
int first_bad_i = -1;
int first_bad_v = 0;
for (int i = 0; i < N; ++i)
{
if (h_arr[i] != 2)
{
++mismatches;
if (first_bad_i < 0)
{
first_bad_i = i;
first_bad_v = h_arr[i];
}
}
}
INFO("iter=" << iter << " mismatches=" << mismatches << " first_bad_idx=" << first_bad_i
<< " first_bad_val=" << first_bad_v);
REQUIRE(mismatches == 0);
}
REQUIRE(cudaFree(d_arr) == cudaSuccess);
REQUIRE(cudaStreamDestroy(s) == cudaSuccess);
}

80
cccl_upstream/c/experimental/stf/test/test_task.cpp Normal file
View File

@@ -0,0 +1,80 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <vector>
#include <cuda_runtime.h>
#include <c2h/catch2_test_helper.h>
#include <cccl/c/experimental/stf/stf.h>
C2H_TEST("empty stf tasks", "[task]")
{
size_t N = 1000000;
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
std::vector<float> X(N);
std::vector<float> Y(N);
std::vector<float> Z(N);
stf_logical_data_handle lX = stf_logical_data(ctx, X.data(), N * sizeof(float));
stf_logical_data_handle lY = stf_logical_data(ctx, Y.data(), N * sizeof(float));
stf_logical_data_handle lZ = stf_logical_data(ctx, Z.data(), N * sizeof(float));
REQUIRE(lX != nullptr);
REQUIRE(lY != nullptr);
REQUIRE(lZ != nullptr);
stf_logical_data_set_symbol(lX, "X");
stf_logical_data_set_symbol(lY, "Y");
stf_logical_data_set_symbol(lZ, "Z");
stf_task_handle t1 = stf_task_create(ctx);
REQUIRE(t1 != nullptr);
stf_task_set_symbol(t1, "T1");
stf_task_add_dep(t1, lX, STF_RW);
stf_task_start(t1);
stf_task_end(t1);
stf_task_destroy(t1);
stf_task_handle t2 = stf_task_create(ctx);
REQUIRE(t2 != nullptr);
stf_task_set_symbol(t2, "T2");
stf_task_add_dep(t2, lX, STF_READ);
stf_task_add_dep(t2, lY, STF_RW);
stf_task_start(t2);
stf_task_end(t2);
stf_task_destroy(t2);
stf_task_handle t3 = stf_task_create(ctx);
REQUIRE(t3 != nullptr);
stf_task_set_symbol(t3, "T3");
stf_task_add_dep(t3, lX, STF_READ);
stf_task_add_dep(t3, lZ, STF_RW);
stf_task_start(t3);
stf_task_end(t3);
stf_task_destroy(t3);
stf_task_handle t4 = stf_task_create(ctx);
REQUIRE(t4 != nullptr);
stf_task_set_symbol(t4, "T4");
stf_task_add_dep(t4, lY, STF_READ);
stf_task_add_dep(t4, lZ, STF_RW);
stf_task_start(t4);
stf_task_end(t4);
stf_task_destroy(t4);
stf_logical_data_destroy(lX);
stf_logical_data_destroy(lY);
stf_logical_data_destroy(lZ);
stf_ctx_finalize(ctx);
}

94
cccl_upstream/c/experimental/stf/test/test_task_get_graph.cu Normal file
View File

@@ -0,0 +1,94 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cmath>
#include <cstdint>
#include <cuda_runtime.h>
#include <c2h/catch2_test_helper.h>
#include <cccl/c/experimental/stf/stf.h>
__global__ void scale_kernel(int cnt, double* data, double factor)
{
const int tid = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
const int nthreads = static_cast<int>(gridDim.x * blockDim.x);
for (int i = tid; i < cnt; i += nthreads)
{
data[i] *= factor;
}
}
// Exercise the explicit-graph path: instead of capturing a stream with
// stf_task_enable_capture() + stf_task_get_custream(), an expert caller fetches
// the task's child cudaGraph_t with stf_task_get_graph() and adds nodes into it
// directly (here a single kernel node). STF wires the task's dependencies around
// the child graph.
C2H_TEST("task_get_graph: explicit kernel node in a stackable graph scope", "[stackable][task_get_graph]")
{
const size_t N = 256;
stf_ctx_handle ctx = stf_stackable_ctx_create();
REQUIRE(ctx != nullptr);
double* host_data;
REQUIRE(cudaMallocHost(&host_data, N * sizeof(double)) == cudaSuccess);
for (size_t i = 0; i < N; i++)
{
host_data[i] = static_cast<double>(i);
}
stf_logical_data_handle lA = stf_stackable_logical_data(ctx, host_data, N * sizeof(double));
REQUIRE(lA != nullptr);
// Multiply by 3 inside a nested graph scope using an explicitly added kernel node.
stf_stackable_push_graph(ctx);
{
stf_task_handle t = stf_stackable_task_create(ctx);
REQUIRE(t != nullptr);
stf_stackable_task_add_dep(ctx, t, lA, STF_RW);
// Note: no stf_task_enable_capture() here -- the explicit-graph path is
// mutually exclusive with stream capture.
stf_task_start(t);
cudaGraph_t g = stf_task_get_graph(t);
REQUIRE(g != nullptr);
double* d = static_cast<double*>(stf_task_get(t, 0));
int n = static_cast<int>(N);
double f = 3.0;
void* kernel_args[] = {&n, &d, &f};
cudaKernelNodeParams kparams = {};
kparams.func = reinterpret_cast<void*>(&scale_kernel);
kparams.gridDim = dim3(2, 1, 1);
kparams.blockDim = dim3(64, 1, 1);
kparams.sharedMemBytes = 0;
kparams.kernelParams = kernel_args;
kparams.extra = nullptr;
cudaGraphNode_t node;
REQUIRE(cudaGraphAddKernelNode(&node, g, nullptr, 0, &kparams) == cudaSuccess);
stf_task_end(t);
stf_task_destroy(t);
}
stf_stackable_pop(ctx);
stf_stackable_logical_data_destroy(lA);
stf_stackable_ctx_finalize(ctx);
for (size_t i = 0; i < N; i++)
{
REQUIRE(std::fabs(host_data[i] - 3.0 * static_cast<double>(i)) < 1e-10);
}
REQUIRE(cudaFreeHost(host_data) == cudaSuccess);
}

72
cccl_upstream/c/experimental/stf/test/test_token.cpp Normal file
View File

@@ -0,0 +1,72 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda_runtime.h>
#include <c2h/catch2_test_helper.h>
#include <cccl/c/experimental/stf/stf.h>
C2H_TEST("stf token", "[token]")
{
stf_ctx_handle ctx = stf_ctx_create();
REQUIRE(ctx != nullptr);
stf_logical_data_handle lX = stf_token(ctx);
stf_logical_data_handle lY = stf_token(ctx);
stf_logical_data_handle lZ = stf_token(ctx);
REQUIRE(lX != nullptr);
REQUIRE(lY != nullptr);
REQUIRE(lZ != nullptr);
stf_logical_data_set_symbol(lX, "X");
stf_logical_data_set_symbol(lY, "Y");
stf_logical_data_set_symbol(lZ, "Z");
stf_task_handle t1 = stf_task_create(ctx);
REQUIRE(t1 != nullptr);
stf_task_set_symbol(t1, "T1");
stf_task_add_dep(t1, lX, STF_RW);
stf_task_start(t1);
stf_task_end(t1);
stf_task_destroy(t1);
stf_task_handle t2 = stf_task_create(ctx);
REQUIRE(t2 != nullptr);
stf_task_set_symbol(t2, "T2");
stf_task_add_dep(t2, lX, STF_READ);
stf_task_add_dep(t2, lY, STF_RW);
stf_task_start(t2);
stf_task_end(t2);
stf_task_destroy(t2);
stf_task_handle t3 = stf_task_create(ctx);
REQUIRE(t3 != nullptr);
stf_task_set_symbol(t3, "T3");
stf_task_add_dep(t3, lX, STF_READ);
stf_task_add_dep(t3, lZ, STF_RW);
stf_task_start(t3);
stf_task_end(t3);
stf_task_destroy(t3);
stf_task_handle t4 = stf_task_create(ctx);
REQUIRE(t4 != nullptr);
stf_task_set_symbol(t4, "T4");
stf_task_add_dep(t4, lY, STF_READ);
stf_task_add_dep(t4, lZ, STF_RW);
stf_task_start(t4);
stf_task_end(t4);
stf_task_destroy(t4);
stf_logical_data_destroy(lX);
stf_logical_data_destroy(lY);
stf_logical_data_destroy(lZ);
stf_ctx_finalize(ctx);
}

120
cccl_upstream/c/parallel.v2/CMakeLists.txt Normal file
View File

@@ -0,0 +1,120 @@
# 3.30 is required for FindCUDAToolkit's CUDA::nvfatbin / CUDA::nvfatbin_static
# imported targets, which the HostJIT linker chain depends on.
cmake_minimum_required(VERSION 3.30)
project(CCCL_C_Parallel_V2 LANGUAGES CUDA CXX C)
# Bootstrap CCCL cmake helpers when building c/parallel.v2 in isolation
# (i.e. not as a subdirectory of the CCCL super-project).
if (NOT COMMAND cccl_configure_target)
# Repo root is two levels up from this file (c/parallel.v2 -> c -> cccl)
get_filename_component(
_cccl_root
"${CMAKE_CURRENT_SOURCE_DIR}/../.."
ABSOLUTE
)
set(CCCL_SOURCE_DIR "${_cccl_root}" CACHE PATH "CCCL repo root" FORCE)
set(
CCCL_BINARY_DIR
"${CMAKE_CURRENT_BINARY_DIR}"
CACHE PATH
"CCCL binary root"
FORCE
)
include("${_cccl_root}/cmake/CCCLUtilities.cmake")
include("${_cccl_root}/cmake/CCCLConfigureTarget.cmake")
include("${_cccl_root}/cmake/CCCLGetDependencies.cmake")
if (NOT TARGET cccl.compiler_interface)
add_library(cccl.compiler_interface INTERFACE)
endif()
endif()
option(CCCL_C_Parallel_V2_ENABLE_TESTING "Build cccl.c.parallel.v2 tests." OFF)
set(
CCCL_C_PARALLEL_V2_LIBRARY_OUTPUT_DIRECTORY
""
CACHE PATH
"Override output directory for the cccl.c.parallel.v2 library"
)
mark_as_advanced(CCCL_C_PARALLEL_V2_LIBRARY_OUTPUT_DIRECTORY)
file(
GLOB_RECURSE srcs
RELATIVE "${CMAKE_CURRENT_LIST_DIR}"
CONFIGURE_DEPENDS
"src/*.cu"
"src/*.cpp"
)
# hostjit sources are built as a separate library
list(FILTER srcs EXCLUDE REGEX "^src/hostjit/")
# Editor lock/temp files
list(FILTER srcs EXCLUDE REGEX "/\\.#")
add_library(cccl.c.parallel.v2 SHARED ${srcs})
set_property(TARGET cccl.c.parallel.v2 PROPERTY POSITION_INDEPENDENT_CODE ON)
cccl_configure_target(cccl.c.parallel.v2 DIALECT 20)
if (CCCL_C_PARALLEL_V2_LIBRARY_OUTPUT_DIRECTORY)
set_target_properties(
cccl.c.parallel.v2
PROPERTIES
LIBRARY_OUTPUT_DIRECTORY "${CCCL_C_PARALLEL_V2_LIBRARY_OUTPUT_DIRECTORY}"
ARCHIVE_OUTPUT_DIRECTORY "${CCCL_C_PARALLEL_V2_LIBRARY_OUTPUT_DIRECTORY}"
RUNTIME_OUTPUT_DIRECTORY "${CCCL_C_PARALLEL_V2_LIBRARY_OUTPUT_DIRECTORY}"
)
endif()
cccl_get_cub()
cccl_get_cudatoolkit()
cccl_get_thrust()
add_subdirectory(src/hostjit)
set_target_properties(cccl.c.parallel.v2 PROPERTIES CUDA_RUNTIME_LIBRARY STATIC)
target_link_libraries(
cccl.c.parallel.v2
PRIVATE
cccl.compiler_interface
CUDA::cudart_static
CUDA::cuda_driver
CUB::CUB
Thrust::Thrust
cccl.c.parallel.v2.hostjit_lib # transitively brings in nvJitLink, nvfatbin, nvptxcompiler
)
if (WIN32)
target_link_libraries(cccl.c.parallel.v2 PRIVATE Dbghelp)
# We are shadowing a lot of variables with the globals like num_items
target_compile_options(
cccl.c.parallel.v2
PRIVATE $<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:-Xcompiler=/wd4459>
)
target_compile_definitions(
cccl.c.parallel.v2
PRIVATE CATCH_CONFIG_NO_WINDOWS_SEH
)
endif()
target_compile_definitions(
cccl.c.parallel.v2
PUBLIC CCCL_C_EXPERIMENTAL=1
PRIVATE #
NVRTC_GET_TYPE_NAME=1
CUB_DISABLE_CDP=1
CUB_DEFINE_RUNTIME_POLICIES
)
target_compile_options(
cccl.c.parallel.v2
PRIVATE $<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:--extended-lambda>
)
target_include_directories(
cccl.c.parallel.v2 #
PUBLIC "include"
PRIVATE "src" "src/hostjit/include"
)
if (CCCL_C_Parallel_V2_ENABLE_TESTING)
add_subdirectory(test)
endif()

82
cccl_upstream/c/parallel.v2/include/cccl/c/binary_search.h Normal file
View File

@@ -0,0 +1,82 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_binary_search_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler; // hostjit::JITCompiler*
#if defined(_WIN32)
// Opaque state for serializing CUB's lazy first-call initialization.
void* first_call_state;
#endif // _WIN32
void* binary_search_fn; // int(*)(void*, ull, void*, ull, void*, void*, void*)
} cccl_device_binary_search_build_result_t;
CCCL_C_API CUresult cccl_device_binary_search_build(
cccl_device_binary_search_build_result_t* build,
cccl_binary_search_mode_t mode,
cccl_iterator_t d_data,
cccl_iterator_t d_values,
cccl_iterator_t d_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_binary_search_build_ex(
cccl_device_binary_search_build_result_t* build,
cccl_binary_search_mode_t mode,
cccl_iterator_t d_data,
cccl_iterator_t d_values,
cccl_iterator_t d_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_binary_search(
cccl_device_binary_search_build_result_t build,
cccl_iterator_t d_data,
uint64_t num_items,
cccl_iterator_t d_values,
uint64_t num_values,
cccl_iterator_t d_out,
cccl_op_t op,
CUstream stream);
CCCL_C_API CUresult cccl_device_binary_search_cleanup(cccl_device_binary_search_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

23
cccl_upstream/c/parallel.v2/include/cccl/c/extern_c.h Normal file
View File

@@ -0,0 +1,23 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
#ifdef __cplusplus
# define CCCL_C_EXTERN_C_BEGIN extern "C" {
# define CCCL_C_EXTERN_C_END }
#else
# define CCCL_C_EXTERN_C_BEGIN
# define CCCL_C_EXTERN_C_END
#endif

65
cccl_upstream/c/parallel.v2/include/cccl/c/for.h Normal file
View File

@@ -0,0 +1,65 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_for_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler; // hostjit::JITCompiler*
void* for_fn; // int(*)(void*, unsigned long long, void*)
} cccl_device_for_build_result_t;
CCCL_C_API CUresult cccl_device_for_build(
cccl_device_for_build_result_t* build,
cccl_iterator_t d_data,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_for_build_ex(
cccl_device_for_build_result_t* build,
cccl_iterator_t d_data,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_for(
cccl_device_for_build_result_t build, cccl_iterator_t d_data, uint64_t num_items, cccl_op_t op, CUstream stream);
CCCL_C_API CUresult cccl_device_for_cleanup(cccl_device_for_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

96
cccl_upstream/c/parallel.v2/include/cccl/c/histogram.h Normal file
View File

@@ -0,0 +1,96 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdbool.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_histogram_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler;
void* histogram_fn;
cccl_type_info counter_type;
cccl_type_info level_type;
cccl_type_info sample_type;
int num_channels;
int num_active_channels;
} cccl_device_histogram_build_result_t;
CCCL_C_API CUresult cccl_device_histogram_build(
cccl_device_histogram_build_result_t* build,
int num_channels,
int num_active_channels,
cccl_iterator_t d_samples,
int num_output_levels_val,
cccl_iterator_t d_output_histograms,
cccl_type_info level_type,
int64_t num_rows,
int64_t row_stride_samples,
bool is_evenly_segmented,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_histogram_build_ex(
cccl_device_histogram_build_result_t* build,
int num_channels,
int num_active_channels,
cccl_iterator_t d_samples,
int num_output_levels_val,
cccl_iterator_t d_output_histograms,
cccl_type_info level_type,
int64_t num_rows,
int64_t row_stride_samples,
bool is_evenly_segmented,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_histogram_even(
cccl_device_histogram_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_samples,
cccl_iterator_t d_output_histograms,
cccl_value_t num_output_levels,
cccl_value_t lower_level,
cccl_value_t upper_level,
int64_t num_row_pixels,
int64_t num_rows,
int64_t row_stride_samples,
CUstream stream);
CCCL_C_API CUresult cccl_device_histogram_cleanup(cccl_device_histogram_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

86
cccl_upstream/c/parallel.v2/include/cccl/c/merge_sort.h Normal file
View File

@@ -0,0 +1,86 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_merge_sort_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler;
void* sort_fn;
// 1 if the build compiled SortKeysCopy (no items), 0 if SortPairsCopy. The
// run function dispatches on this so the value-vs-pairs decision doesn't
// have to be re-derived from the iterator arguments.
int keys_only;
cccl_type_info key_type;
cccl_type_info item_type;
} cccl_device_merge_sort_build_result_t;
CCCL_C_API CUresult cccl_device_merge_sort_build(
cccl_device_merge_sort_build_result_t* build,
cccl_iterator_t d_in_keys,
cccl_iterator_t d_in_items,
cccl_iterator_t d_out_keys,
cccl_iterator_t d_out_items,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_merge_sort_build_ex(
cccl_device_merge_sort_build_result_t* build,
cccl_iterator_t d_in_keys,
cccl_iterator_t d_in_items,
cccl_iterator_t d_out_keys,
cccl_iterator_t d_out_items,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_merge_sort(
cccl_device_merge_sort_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in_keys,
cccl_iterator_t d_in_items,
cccl_iterator_t d_out_keys,
cccl_iterator_t d_out_items,
uint64_t num_items,
cccl_op_t op,
CUstream stream);
CCCL_C_API CUresult cccl_device_merge_sort_cleanup(cccl_device_merge_sort_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

90
cccl_upstream/c/parallel.v2/include/cccl/c/radix_sort.h Normal file
View File

@@ -0,0 +1,90 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdbool.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_radix_sort_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler; /* Owns both wrappers below — one TU, one cubin */
void* sort_fn; /* Wrapper around CUB's copy-overload (selector always 0) */
void* sort_fn_overwrite; /* Wrapper around CUB's DoubleBuffer overload; reports selector */
cccl_type_info key_type;
cccl_type_info value_type;
cccl_sort_order_t order;
int keys_only; /* 1 if keys-only sort, 0 if key-value pairs */
} cccl_device_radix_sort_build_result_t;
CCCL_C_API CUresult cccl_device_radix_sort_build(
cccl_device_radix_sort_build_result_t* build,
cccl_sort_order_t sort_order,
cccl_iterator_t input_keys_it,
cccl_iterator_t input_values_it,
cccl_op_t decomposer,
const char* decomposer_return_type,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_radix_sort_build_ex(
cccl_device_radix_sort_build_result_t* build,
cccl_sort_order_t sort_order,
cccl_iterator_t input_keys_it,
cccl_iterator_t input_values_it,
cccl_op_t decomposer,
const char* decomposer_return_type,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_radix_sort(
cccl_device_radix_sort_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_keys_out,
cccl_iterator_t d_values_in,
cccl_iterator_t d_values_out,
cccl_op_t decomposer,
uint64_t num_items,
int begin_bit,
int end_bit,
bool is_overwrite_okay,
int* selector,
CUstream stream);
CCCL_C_API CUresult cccl_device_radix_sort_cleanup(cccl_device_radix_sort_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

94
cccl_upstream/c/parallel.v2/include/cccl/c/reduce.h Normal file
View File

@@ -0,0 +1,94 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_reduce_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler; // hostjit::JITCompiler*
void* reduce_fn; // int(*)(void*, size_t*, void*, void*, unsigned long long, void*, void*, void*) — trailing void* is
// the CUstream
uint64_t accumulator_size;
cccl_determinism_t determinism;
} cccl_device_reduce_build_result_t;
// TODO return a union of nvtx/cuda/nvrtc errors or a string?
CCCL_C_API CUresult cccl_device_reduce_build(
cccl_device_reduce_build_result_t* build,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_op_t op,
cccl_value_t init,
cccl_determinism_t determinism,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_reduce_build_ex(
cccl_device_reduce_build_result_t* build,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_op_t op,
cccl_value_t init,
cccl_determinism_t determinism,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_reduce(
cccl_device_reduce_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
cccl_value_t init,
CUstream stream);
CCCL_C_API CUresult cccl_device_reduce_nondeterministic(
cccl_device_reduce_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
cccl_value_t init,
CUstream stream);
CCCL_C_API CUresult cccl_device_reduce_cleanup(cccl_device_reduce_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

127
cccl_upstream/c/parallel.v2/include/cccl/c/scan.h Normal file
View File

@@ -0,0 +1,127 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdbool.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_scan_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler;
void* scan_fn;
bool force_inclusive;
cccl_init_kind_t init_kind;
} cccl_device_scan_build_result_t;
CCCL_C_API CUresult cccl_device_scan_build(
cccl_device_scan_build_result_t* build_ptr,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_op_t op,
cccl_type_info init,
bool force_inclusive,
cccl_init_kind_t init_kind,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_scan_build_ex(
cccl_device_scan_build_result_t* build_ptr,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_op_t op,
cccl_type_info init,
bool force_inclusive,
cccl_init_kind_t init_kind,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_exclusive_scan(
cccl_device_scan_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
cccl_value_t init,
CUstream stream);
CCCL_C_API CUresult cccl_device_inclusive_scan(
cccl_device_scan_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
cccl_value_t init,
CUstream stream);
CCCL_C_API CUresult cccl_device_exclusive_scan_future_value(
cccl_device_scan_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
cccl_iterator_t init,
CUstream stream);
CCCL_C_API CUresult cccl_device_inclusive_scan_future_value(
cccl_device_scan_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
cccl_iterator_t init,
CUstream stream);
CCCL_C_API CUresult cccl_device_inclusive_scan_no_init(
cccl_device_scan_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
CUstream stream);
CCCL_C_API CUresult cccl_device_scan_cleanup(cccl_device_scan_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

84
cccl_upstream/c/parallel.v2/include/cccl/c/segmented_reduce.h Normal file
View File

@@ -0,0 +1,84 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_segmented_reduce_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler;
void* segmented_reduce_fn;
} cccl_device_segmented_reduce_build_result_t;
// TODO return a union of nvtx/cuda/nvrtc errors or a string?
CCCL_C_API CUresult cccl_device_segmented_reduce_build(
cccl_device_segmented_reduce_build_result_t* build,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_iterator_t begin_offset_in,
cccl_iterator_t end_offset_in,
cccl_op_t op,
cccl_value_t init,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_segmented_reduce_build_ex(
cccl_device_segmented_reduce_build_result_t* build,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_iterator_t begin_offset_in,
cccl_iterator_t end_offset_in,
cccl_op_t op,
cccl_value_t init,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_segmented_reduce(
cccl_device_segmented_reduce_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_segments,
cccl_iterator_t begin_offset_in,
cccl_iterator_t end_offset_in,
cccl_op_t op,
cccl_value_t init,
CUstream stream);
CCCL_C_API CUresult cccl_device_segmented_reduce_cleanup(cccl_device_segmented_reduce_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

91
cccl_upstream/c/parallel.v2/include/cccl/c/segmented_sort.h Normal file
View File

@@ -0,0 +1,91 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdbool.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_segmented_sort_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler; /* Owns both wrappers below — one TU, one cubin */
void* sort_fn; /* Wrapper around CUB's copy-overload (selector always 0) */
void* sort_fn_overwrite; /* Wrapper around CUB's DoubleBuffer overload; reports selector */
cccl_type_info key_type;
cccl_type_info value_type;
cccl_sort_order_t order;
int keys_only; /* 1 if keys-only sort, 0 if key-value pairs */
} cccl_device_segmented_sort_build_result_t;
// TODO return a union of nvtx/cuda/nvrtc errors or a string?
CCCL_C_API CUresult cccl_device_segmented_sort_build(
cccl_device_segmented_sort_build_result_t* build,
cccl_sort_order_t sort_order,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_values_in,
cccl_iterator_t begin_offset_in,
cccl_iterator_t end_offset_in,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_segmented_sort_build_ex(
cccl_device_segmented_sort_build_result_t* build,
cccl_sort_order_t sort_order,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_values_in,
cccl_iterator_t begin_offset_in,
cccl_iterator_t end_offset_in,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_segmented_sort(
cccl_device_segmented_sort_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_keys_out,
cccl_iterator_t d_values_in,
cccl_iterator_t d_values_out,
uint64_t num_items,
uint64_t num_segments,
cccl_iterator_t begin_offset_in,
cccl_iterator_t end_offset_in,
bool is_overwrite_okay,
int* selector,
CUstream stream);
CCCL_C_API CUresult cccl_device_segmented_sort_cleanup(cccl_device_segmented_sort_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

87
cccl_upstream/c/parallel.v2/include/cccl/c/three_way_partition.h Normal file
View File

@@ -0,0 +1,87 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_three_way_partition_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler;
void* three_way_partition_fn;
} cccl_device_three_way_partition_build_result_t;
// TODO return a union of nvtx/cuda/nvrtc errors or a string?
CCCL_C_API CUresult cccl_device_three_way_partition_build(
cccl_device_three_way_partition_build_result_t* build,
cccl_iterator_t d_in,
cccl_iterator_t d_first_part_out,
cccl_iterator_t d_second_part_out,
cccl_iterator_t d_unselected_out,
cccl_iterator_t d_num_selected_out,
cccl_op_t select_first_part_op,
cccl_op_t select_second_part_op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_three_way_partition_build_ex(
cccl_device_three_way_partition_build_result_t* build,
cccl_iterator_t d_in,
cccl_iterator_t d_first_part_out,
cccl_iterator_t d_second_part_out,
cccl_iterator_t d_unselected_out,
cccl_iterator_t d_num_selected_out,
cccl_op_t select_first_part_op,
cccl_op_t select_second_part_op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_three_way_partition(
cccl_device_three_way_partition_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_first_part_out,
cccl_iterator_t d_second_part_out,
cccl_iterator_t d_unselected_out,
cccl_iterator_t d_num_selected_out,
cccl_op_t select_first_part_op,
cccl_op_t select_second_part_op,
uint64_t num_items,
CUstream stream);
CCCL_C_API CUresult cccl_device_three_way_partition_cleanup(cccl_device_three_way_partition_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

113
cccl_upstream/c/parallel.v2/include/cccl/c/transform.h Normal file
View File

@@ -0,0 +1,113 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_transform_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler;
#if defined(_WIN32)
// Opaque state for serializing CUB's lazy first-call initialization.
void* first_call_state;
#endif // _WIN32
void* transform_fn;
} cccl_device_transform_build_result_t;
CCCL_C_API CUresult cccl_device_unary_transform_build(
cccl_device_transform_build_result_t* build_ptr,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_unary_transform_build_ex(
cccl_device_transform_build_result_t* build_ptr,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_unary_transform(
cccl_device_transform_build_result_t build,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
CUstream stream);
CCCL_C_API CUresult cccl_device_binary_transform_build(
cccl_device_transform_build_result_t* build_ptr,
cccl_iterator_t d_in1,
cccl_iterator_t d_in2,
cccl_iterator_t d_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_binary_transform_build_ex(
cccl_device_transform_build_result_t* build_ptr,
cccl_iterator_t d_in1,
cccl_iterator_t d_in2,
cccl_iterator_t d_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_binary_transform(
cccl_device_transform_build_result_t build,
cccl_iterator_t d_in1,
cccl_iterator_t d_in2,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
CUstream stream);
CCCL_C_API CUresult cccl_device_transform_cleanup(cccl_device_transform_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

188
cccl_upstream/c/parallel.v2/include/cccl/c/types.h Normal file
View File

@@ -0,0 +1,188 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#if defined(_WIN32)
# define CCCL_C_API __declspec(dllexport)
#else // ^^^ _WIN32 ^^^ / vvv !_WIN32 vvv
# define CCCL_C_API __attribute__((__visibility__("default")))
#endif // !_WIN32
#include <stddef.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
CCCL_C_EXTERN_C_BEGIN
typedef enum cccl_type_enum
{
CCCL_INT8 = 0,
CCCL_INT16 = 1,
CCCL_INT32 = 2,
CCCL_INT64 = 3,
CCCL_UINT8 = 4,
CCCL_UINT16 = 5,
CCCL_UINT32 = 6,
CCCL_UINT64 = 7,
CCCL_FLOAT16 = 8, // This may be unsupported if _CCCL_HAS_NVFP16() is false but we can't include the header to check
// that here
CCCL_FLOAT32 = 9,
CCCL_FLOAT64 = 10,
CCCL_STORAGE = 11,
CCCL_BOOLEAN = 12,
} cccl_type_enum;
typedef struct cccl_type_info
{
size_t size;
size_t alignment;
cccl_type_enum type;
} cccl_type_info;
typedef enum cccl_op_kind_t
{
// Arbitrary semantics, without state.
CCCL_STATELESS = 0,
// Arbitrary semantics, with state.
CCCL_STATEFUL = 1,
// Well-known semantics, required to be stateless.
// Equivalent to corresponding function objects in C++'s <functional>.
// If the types involved are primitive, only the kind field is necessary.
// Otherwise, the cccl_op_t object must also contain the rest of the fields,
// as appropriate.
CCCL_PLUS = 2,
CCCL_MINUS = 3,
CCCL_MULTIPLIES = 4,
CCCL_DIVIDES = 5,
CCCL_MODULUS = 6,
CCCL_EQUAL_TO = 7,
CCCL_NOT_EQUAL_TO = 8,
CCCL_GREATER = 9,
CCCL_LESS = 10,
CCCL_GREATER_EQUAL = 11,
CCCL_LESS_EQUAL = 12,
CCCL_LOGICAL_AND = 13,
CCCL_LOGICAL_OR = 14,
CCCL_LOGICAL_NOT = 15,
CCCL_BIT_AND = 16,
CCCL_BIT_OR = 17,
CCCL_BIT_XOR = 18,
CCCL_BIT_NOT = 19,
CCCL_IDENTITY = 20,
CCCL_NEGATE = 21,
CCCL_MINIMUM = 22,
CCCL_MAXIMUM = 23,
} cccl_op_kind_t;
typedef enum cccl_op_code_type
{
CCCL_OP_LTOIR = 0, // Pre-compiled LTO-IR (escape hatch for callers with existing nvcc -dlto artifacts).
// LTO-IR is a binary container passed to nvJitLink at the PTX level — the LLVM optimizer
// never sees it, so the operator cannot be inlined into the CUB kernel and pays a real
// CALL on every iteration. CCCL_OP_LLVM_IR feeds LLVM's bitcode linker instead, which
// merges the operator into the CUB module before PTX codegen and enables full inlining.
// Prefer CCCL_OP_LLVM_IR or CCCL_OP_CPP_SOURCE for any new code.
CCCL_OP_CPP_SOURCE = 1, // C++ source code (compiled to LLVM bitcode by hostjit's Clang).
CCCL_OP_LLVM_IR = 2 // LLVM bitcode (recommended) — merges into the CUB module before PTX gen, so inlines.
} cccl_op_code_type;
typedef struct cccl_op_t
{
cccl_op_kind_t type;
const char* name;
const char* code;
size_t code_size;
cccl_op_code_type code_type;
size_t size;
size_t alignment;
void* state;
const char** extra_ltoirs;
size_t* extra_ltoir_sizes;
size_t num_extra_ltoirs;
cccl_op_code_type* extra_code_types;
} cccl_op_t;
typedef struct cccl_build_config
{
const char** extra_compile_flags; // e.g., {"-DENABLE_FAST_MATH", "-O3"}
size_t num_extra_compile_flags;
const char** extra_include_dirs; // e.g., {"/path/to/my/headers"}
size_t num_extra_include_dirs;
int enable_pch; // Cache precompiled headers on disk to speed up repeated builds
int verbose; // Log PCH generation/usage and compiler args to build diagnostics
} cccl_build_config;
typedef enum cccl_iterator_kind_t
{
CCCL_POINTER = 0,
CCCL_ITERATOR = 1,
} cccl_iterator_kind_t;
typedef struct cccl_value_t
{
cccl_type_info type;
void* state;
} cccl_value_t;
typedef union
{
int64_t signed_offset;
uint64_t unsigned_offset;
} cccl_increment_t;
typedef void (*cccl_host_op_fn_ptr_t)(void*, cccl_increment_t);
typedef struct cccl_iterator_t
{
size_t size;
size_t alignment;
cccl_iterator_kind_t type;
cccl_op_t advance;
cccl_op_t dereference;
cccl_type_info value_type;
void* state;
cccl_host_op_fn_ptr_t host_advance;
} cccl_iterator_t;
typedef enum cccl_sort_order_t
{
CCCL_ASCENDING = 0,
CCCL_DESCENDING = 1,
} cccl_sort_order_t;
typedef enum cccl_init_kind_t
{
CCCL_VALUE_INIT = 0,
CCCL_FUTURE_VALUE_INIT = 1,
CCCL_NO_INIT = 2,
} cccl_init_kind_t;
typedef enum cccl_determinism_t
{
CCCL_NOT_GUARANTEED = 0,
CCCL_RUN_TO_RUN = 1,
CCCL_GPU_TO_GPU = 2,
} cccl_determinism_t;
typedef enum cccl_binary_search_mode_t
{
CCCL_BINARY_SEARCH_LOWER_BOUND = 0,
CCCL_BINARY_SEARCH_UPPER_BOUND = 1,
} cccl_binary_search_mode_t;
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

83
cccl_upstream/c/parallel.v2/include/cccl/c/unique_by_key.h Normal file
View File

@@ -0,0 +1,83 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_unique_by_key_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler;
void* unique_by_key_fn;
} cccl_device_unique_by_key_build_result_t;
CCCL_C_API CUresult cccl_device_unique_by_key_build(
cccl_device_unique_by_key_build_result_t* build,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_values_in,
cccl_iterator_t d_keys_out,
cccl_iterator_t d_values_out,
cccl_iterator_t d_num_selected_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_unique_by_key_build_ex(
cccl_device_unique_by_key_build_result_t* build,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_values_in,
cccl_iterator_t d_keys_out,
cccl_iterator_t d_values_out,
cccl_iterator_t d_num_selected_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_unique_by_key(
cccl_device_unique_by_key_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_values_in,
cccl_iterator_t d_keys_out,
cccl_iterator_t d_values_out,
cccl_iterator_t d_num_selected_out,
cccl_op_t op,
uint64_t num_items,
CUstream stream);
CCCL_C_API CUresult cccl_device_unique_by_key_cleanup(cccl_device_unique_by_key_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

180
cccl_upstream/c/parallel.v2/src/binary_search.cu Normal file
View File

@@ -0,0 +1,180 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#include <cuda/std/version>
#include <cstdio>
#include <cstring>
#include <memory>
#include <cccl/c/binary_search.h>
#include <hostjit/codegen/cub_call.hpp>
#include <util/build_utils.h>
#include <util/first_call_gate.h>
using namespace hostjit::codegen;
// (d_in_0, num_items, d_in_1, num_values, d_out_0, op_0_state, stream)
using binary_search_fn_t = int (*)(void*, unsigned long long, void*, unsigned long long, void*, void*, void*);
CUresult cccl_device_binary_search_build_ex(
cccl_device_binary_search_build_result_t* build_ptr,
cccl_binary_search_mode_t mode,
cccl_iterator_t d_data,
cccl_iterator_t d_values,
cccl_iterator_t d_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config)
try
{
if (build_ptr == nullptr)
{
return CUDA_ERROR_INVALID_VALUE;
}
#if CCCL_OS(WINDOWS)
build_ptr->first_call_state = nullptr;
#endif
const std::string cccl_include_str = cccl::detail::parse_cccl_include_path(libcudacxx_path);
const std::string ctk_root_str = cccl::detail::parse_ctk_root(ctk_path);
const char* const cccl_include_path = cccl_include_str.empty() ? nullptr : cccl_include_str.c_str();
const char* const ctk_root = ctk_root_str.empty() ? nullptr : ctk_root_str.c_str();
cccl::detail::MergedBuildConfig merged(config, cub_path, thrust_path);
#if CCCL_OS(WINDOWS)
auto first_call_state = std::make_unique<cccl::detail::first_call_gate>();
#endif
const char* find_fn =
(mode == CCCL_BINARY_SEARCH_LOWER_BOUND) ? "cub::DeviceFind::LowerBound" : "cub::DeviceFind::UpperBound";
// env_stream uses the env-based DeviceFind overload so CUB manages its own
// temp storage via the env's memory_resource — no caller-managed buffer.
auto result =
CubCall::from("cub/device/device_find.cuh")
.run(find_fn)
.name("cccl_jit_binary_search")
.with(in(d_data), num_haystack, in(d_values), num_needles, out(d_out), cmp(op), env_stream)
.compile(cc_major, cc_minor, merged.get(), ctk_root, cccl_include_path);
build_ptr->cc = cc_major * 10 + cc_minor;
cccl::detail::copy_cubin(result.cubin, build_ptr->payload, build_ptr->payload_size);
build_ptr->jit_compiler = result.compiler;
#if CCCL_OS(WINDOWS)
build_ptr->first_call_state = first_call_state.release();
#endif
build_ptr->binary_search_fn = result.fn_ptr;
return CUDA_SUCCESS;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_binary_search_build_ex(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}
CUresult cccl_device_binary_search(
cccl_device_binary_search_build_result_t build,
cccl_iterator_t d_data,
uint64_t num_items,
cccl_iterator_t d_values,
uint64_t num_values,
cccl_iterator_t d_out,
cccl_op_t op,
CUstream stream)
try
{
if (!build.binary_search_fn)
{
return CUDA_ERROR_INVALID_VALUE;
}
const auto fn = reinterpret_cast<binary_search_fn_t>(build.binary_search_fn);
#if CCCL_OS(WINDOWS)
if (!build.first_call_state)
{
return CUDA_ERROR_INVALID_VALUE;
}
const auto invoke = [&] {
return fn(
d_data.state, num_items, d_values.state, num_values, d_out.state, op.state, reinterpret_cast<void*>(stream));
};
// Empty calls return before DeviceTransform initializes its static launch configuration,
// so they must not complete the first-call gate.
const int status =
num_values == 0 ? invoke() : static_cast<cccl::detail::first_call_gate*>(build.first_call_state)->invoke(invoke);
#else
const int status =
fn(d_data.state, num_items, d_values.state, num_values, d_out.state, op.state, reinterpret_cast<void*>(stream));
#endif
return (status == 0) ? CUDA_SUCCESS : CUDA_ERROR_UNKNOWN;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_binary_search(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}
CUresult cccl_device_binary_search_build(
cccl_device_binary_search_build_result_t* build,
cccl_binary_search_mode_t mode,
cccl_iterator_t d_data,
cccl_iterator_t d_values,
cccl_iterator_t d_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path)
{
return cccl_device_binary_search_build_ex(
build,
mode,
d_data,
d_values,
d_out,
op,
cc_major,
cc_minor,
cub_path,
thrust_path,
libcudacxx_path,
ctk_path,
nullptr);
}
CUresult cccl_device_binary_search_cleanup(cccl_device_binary_search_build_result_t* build_ptr)
try
{
if (build_ptr == nullptr)
{
return CUDA_ERROR_INVALID_VALUE;
}
#if CCCL_OS(WINDOWS)
delete static_cast<cccl::detail::first_call_gate*>(build_ptr->first_call_state);
build_ptr->first_call_state = nullptr;
#endif
cccl::detail::release_jit_artifacts(build_ptr);
build_ptr->binary_search_fn = nullptr;
return CUDA_SUCCESS;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_binary_search_cleanup(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}

117
cccl_upstream/c/parallel.v2/src/for.cu Normal file
View File

@@ -0,0 +1,117 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#include <cstdio>
#include <cstring>
#include <cccl/c/for.h>
#include <hostjit/codegen/cub_call.hpp>
#include <util/build_utils.h>
using namespace hostjit::codegen;
// d_in_0, num_items, op_0_state, stream
using for_fn_t = int (*)(void*, unsigned long long, void*, void*);
CUresult cccl_device_for_build_ex(
cccl_device_for_build_result_t* build_ptr,
cccl_iterator_t d_data,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config)
try
{
if (build_ptr == nullptr)
{
return CUDA_ERROR_INVALID_VALUE;
}
std::string cccl_include_str = cccl::detail::parse_cccl_include_path(libcudacxx_path);
std::string ctk_root_str = cccl::detail::parse_ctk_root(ctk_path);
const char* cccl_include_path = cccl_include_str.empty() ? nullptr : cccl_include_str.c_str();
const char* ctk_root = ctk_root_str.empty() ? nullptr : ctk_root_str.c_str();
cccl::detail::MergedBuildConfig merged(config, cub_path, thrust_path);
auto result =
CubCall::from("cub/device/device_for.cuh")
.run("cub::DeviceFor::ForEachN")
.name("cccl_jit_for")
.with(in(d_data), num_items, for_each_op(op), stream)
.compile(cc_major, cc_minor, merged.get(), ctk_root, cccl_include_path);
build_ptr->cc = cc_major * 10 + cc_minor;
cccl::detail::copy_cubin(result.cubin, build_ptr->payload, build_ptr->payload_size);
build_ptr->jit_compiler = result.compiler;
build_ptr->for_fn = result.fn_ptr;
return CUDA_SUCCESS;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_for_build_ex(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}
CUresult cccl_device_for(
cccl_device_for_build_result_t build, cccl_iterator_t d_data, uint64_t num_items, cccl_op_t op, CUstream stream)
try
{
if (!build.for_fn)
{
return CUDA_ERROR_INVALID_VALUE;
}
auto fn = reinterpret_cast<for_fn_t>(build.for_fn);
const int status = fn(d_data.state, num_items, op.state, reinterpret_cast<void*>(stream));
return (status == 0) ? CUDA_SUCCESS : CUDA_ERROR_UNKNOWN;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_for(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}
CUresult cccl_device_for_build(
cccl_device_for_build_result_t* build,
cccl_iterator_t d_data,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path)
{
return cccl_device_for_build_ex(
build, d_data, op, cc_major, cc_minor, cub_path, thrust_path, libcudacxx_path, ctk_path, nullptr);
}
CUresult cccl_device_for_cleanup(cccl_device_for_build_result_t* build_ptr)
try
{
if (build_ptr == nullptr)
{
return CUDA_ERROR_INVALID_VALUE;
}
cccl::detail::release_jit_artifacts(build_ptr);
build_ptr->for_fn = nullptr;
return CUDA_SUCCESS;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_for_cleanup(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}

209
cccl_upstream/c/parallel.v2/src/histogram.cu Normal file
View File

@@ -0,0 +1,209 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#include <cstdio>
#include <cccl/c/histogram.h>
#include <hostjit/codegen/cub_call.hpp>
#include <util/build_utils.h>
using namespace hostjit::codegen;
// JIT wrapper produced by CubCall:
// fn(temp, temp_bytes,
// d_samples, // input iterator state
// d_histogram, // output pointer (counter_t*)
// &num_levels, // int (host pointer)
// &lower_level, &upper_level, // level_t (host pointer)
// &num_row_pixels, // long long (host pointer)
// &num_rows, // long long (host pointer)
// &row_stride_bytes, // size_t (host-precomputed: row_stride_samples * sizeof(sample_t))
// stream)
using histogram_fn_t = int (*)(void*, size_t*, void*, void*, void*, void*, void*, void*, void*, void*, void*);
static constexpr cccl_type_info k_int_type{sizeof(int), alignof(int), CCCL_INT32};
static constexpr cccl_type_info k_int64_type{sizeof(long long), alignof(long long), CCCL_INT64};
static constexpr cccl_type_info k_size_type{sizeof(unsigned long long), alignof(unsigned long long), CCCL_UINT64};
CUresult cccl_device_histogram_build_ex(
cccl_device_histogram_build_result_t* build_ptr,
int num_channels,
int num_active_channels,
cccl_iterator_t d_samples,
int /*num_output_levels_val*/,
cccl_iterator_t d_output_histograms,
cccl_type_info level_type,
int64_t /*num_rows*/,
int64_t /*row_stride_samples*/,
bool /*is_evenly_segmented*/,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config)
try
{
if (num_channels != 1 || num_active_channels != 1)
{
fprintf(stderr,
"\nERROR in cccl_device_histogram_build(): only num_channels=1, num_active_channels=1 is "
"supported in the HostJIT path.\n");
return CUDA_ERROR_UNKNOWN;
}
std::string cccl_include_str = cccl::detail::parse_cccl_include_path(libcudacxx_path);
std::string ctk_root_str = cccl::detail::parse_ctk_root(ctk_path);
const char* cccl_include_path = cccl_include_str.empty() ? nullptr : cccl_include_str.c_str();
const char* ctk_root = ctk_root_str.empty() ? nullptr : ctk_root_str.c_str();
cccl::detail::MergedBuildConfig merged(config, cub_path, thrust_path);
// level_t comes from the build-time type info. CUB infers
// sample_t / counter_t from the iterator and output pointer respectively.
CubCallResult result =
CubCall::from("cub/device/device_histogram.cuh")
.run("cub::DeviceHistogram::HistogramEven")
.name("cccl_jit_histogram_even")
.with(temp_storage,
temp_bytes,
in(d_samples),
out(d_output_histograms),
typed_scalar(k_int_type, "num_levels"),
typed_scalar(level_type, "lower_level"),
typed_scalar(level_type, "upper_level"),
typed_scalar(k_int64_type, "num_row_pixels"),
typed_scalar(k_int64_type, "num_rows"),
typed_scalar(k_size_type, "row_stride_bytes"),
stream)
.compile(cc_major, cc_minor, merged.get(), ctk_root, cccl_include_path);
build_ptr->cc = cc_major * 10 + cc_minor;
cccl::detail::copy_cubin(result.cubin, build_ptr->payload, build_ptr->payload_size);
build_ptr->jit_compiler = result.compiler;
build_ptr->histogram_fn = result.fn_ptr;
build_ptr->counter_type = d_output_histograms.value_type;
build_ptr->level_type = level_type;
build_ptr->sample_type = d_samples.value_type;
build_ptr->num_channels = num_channels;
build_ptr->num_active_channels = num_active_channels;
return CUDA_SUCCESS;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_histogram_build(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}
CUresult cccl_device_histogram_build(
cccl_device_histogram_build_result_t* build,
int num_channels,
int num_active_channels,
cccl_iterator_t d_samples,
int num_output_levels_val,
cccl_iterator_t d_output_histograms,
cccl_type_info level_type,
int64_t num_rows,
int64_t row_stride_samples,
bool is_evenly_segmented,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path)
{
return cccl_device_histogram_build_ex(
build,
num_channels,
num_active_channels,
d_samples,
num_output_levels_val,
d_output_histograms,
level_type,
num_rows,
row_stride_samples,
is_evenly_segmented,
cc_major,
cc_minor,
cub_path,
thrust_path,
libcudacxx_path,
ctk_path,
nullptr);
}
CUresult cccl_device_histogram_even(
cccl_device_histogram_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_samples,
cccl_iterator_t d_output_histograms,
cccl_value_t num_output_levels,
cccl_value_t lower_level,
cccl_value_t upper_level,
int64_t num_row_pixels,
int64_t num_rows,
int64_t row_stride_samples,
CUstream stream)
try
{
if (!build.histogram_fn)
{
return CUDA_ERROR_INVALID_VALUE;
}
// CUB takes row_stride_bytes (not samples). Pre-compute on the host so the
// JIT wrapper doesn't need a sizeof(sample_t) computation.
long long num_row_pixels_ll = static_cast<long long>(num_row_pixels);
long long num_rows_ll = static_cast<long long>(num_rows);
size_t row_stride_bytes = static_cast<size_t>(row_stride_samples) * build.sample_type.size;
auto fn = reinterpret_cast<histogram_fn_t>(build.histogram_fn);
const int status = fn(
d_temp_storage,
temp_storage_bytes,
d_samples.state,
d_output_histograms.state,
num_output_levels.state,
lower_level.state,
upper_level.state,
&num_row_pixels_ll,
&num_rows_ll,
&row_stride_bytes,
reinterpret_cast<void*>(stream));
return (status == 0) ? CUDA_SUCCESS : CUDA_ERROR_UNKNOWN;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_histogram_even(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}
CUresult cccl_device_histogram_cleanup(cccl_device_histogram_build_result_t* build_ptr)
try
{
if (build_ptr == nullptr)
{
return CUDA_ERROR_INVALID_VALUE;
}
cccl::detail::release_jit_artifacts(build_ptr);
build_ptr->histogram_fn = nullptr;
return CUDA_SUCCESS;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_histogram_cleanup(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}

283
cccl_upstream/c/parallel.v2/src/hostjit/CMakeLists.txt Normal file
View File

@@ -0,0 +1,283 @@
cmake_minimum_required(VERSION 3.30)
# --------------------------------------------------------------------------
# LLVM/Clang/LLD — fetched via CPM as static libraries
# --------------------------------------------------------------------------
# CPM.cmake is at the cccl repo root: cccl/cmake/CPM.cmake
# From c/parallel.v2/src/hostjit/ that's ../../../../cmake/CPM.cmake
set(_cccl_cmake_dir "${CMAKE_CURRENT_SOURCE_DIR}/../../../../cmake")
if (EXISTS "${_cccl_cmake_dir}/CPM.cmake")
include("${_cccl_cmake_dir}/CPM.cmake")
else()
message(FATAL_ERROR "CPM.cmake not found at ${_cccl_cmake_dir}/CPM.cmake")
endif()
if (MSVC AND CMAKE_BUILD_TYPE STREQUAL "Debug")
message(
FATAL_ERROR
"hostjit does not support Debug builds on Windows. "
"The statically-linked LLVM Debug build is too large and causes stack "
"overflows at runtime. Use MinSizeRel, Release, or RelWithDebInfo instead."
)
endif()
set(HOSTJIT_LLVM_VERSION "llvmorg-22.1.1" CACHE STRING "LLVM git tag to fetch")
# List options must be set before CPMAddPackage
set(LLVM_ENABLE_PROJECTS "clang;lld" CACHE STRING "" FORCE)
set(LLVM_TARGETS_TO_BUILD "X86;NVPTX" CACHE STRING "" FORCE)
CPMAddPackage(
NAME llvm_project
GIT_REPOSITORY https://github.com/llvm/llvm-project.git
GIT_TAG ${HOSTJIT_LLVM_VERSION}
GIT_SHALLOW ON
SOURCE_SUBDIR llvm
EXCLUDE_FROM_ALL YES
OPTIONS
"LLVM_BUILD_LLVM_C_DYLIB OFF"
"LLVM_BUILD_TOOLS OFF"
"LLVM_BUILD_UTILS OFF"
"LLVM_BUILD_RUNTIME OFF"
"LLVM_BUILD_RUNTIMES OFF"
"LLVM_INCLUDE_BENCHMARKS OFF"
"LLVM_INCLUDE_DOCS OFF"
"LLVM_INCLUDE_EXAMPLES OFF"
"LLVM_INCLUDE_RUNTIMES OFF"
"LLVM_INCLUDE_TESTS OFF"
"LLVM_INCLUDE_TOOLS ON"
"LLVM_INCLUDE_UTILS OFF"
"LLVM_ENABLE_ZLIB OFF"
"LLVM_ENABLE_ZSTD OFF"
"LLVM_ENABLE_TERMINFO OFF"
"LLVM_ENABLE_BINDINGS OFF"
"CLANG_BUILD_TOOLS OFF"
"CLANG_ENABLE_ARCMT OFF"
"CLANG_ENABLE_STATIC_ANALYZER OFF"
)
# Ensure the clang resource directory exists
file(
MAKE_DIRECTORY "${llvm_project_BINARY_DIR}/lib/clang/${LLVM_VERSION_MAJOR}"
)
# Find CUDA toolkit (may already be found by parent)
if (NOT CUDAToolkit_FOUND)
find_package(CUDAToolkit)
endif()
# --------------------------------------------------------------------------
# hostjit library
# --------------------------------------------------------------------------
add_library(
cccl.c.parallel.v2.hostjit_lib
compiler.cpp
config.cpp
loader.cpp
jit_compiler.cpp
codegen/types.cpp
codegen/iterators.cpp
codegen/operators.cpp
codegen/bitcode.cpp
codegen/cub_call.cpp
)
# CCCL_SOURCE_DIR points to the cccl repo root
# From c/parallel.v2/src/hostjit -> c/parallel.v2/src -> c/parallel.v2 -> c -> cccl
cmake_path(GET CMAKE_CURRENT_SOURCE_DIR PARENT_PATH _src_dir) # c/parallel.v2/src
cmake_path(GET _src_dir PARENT_PATH _c_parallel_dir) # c/parallel.v2
cmake_path(GET _c_parallel_dir PARENT_PATH _c_dir) # c
cmake_path(GET _c_dir PARENT_PATH _cccl_root) # cccl
target_include_directories(
cccl.c.parallel.v2.hostjit_lib
PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}/include
${_c_parallel_dir}/include
${llvm_project_SOURCE_DIR}/llvm/include
${llvm_project_BINARY_DIR}/include
${llvm_project_SOURCE_DIR}/clang/include
${llvm_project_BINARY_DIR}/tools/clang/include
${llvm_project_SOURCE_DIR}/lld/include
${llvm_project_BINARY_DIR}/tools/lld/include
)
target_compile_definitions(
cccl.c.parallel.v2.hostjit_lib
PRIVATE
CCCL_C_EXPERIMENTAL=1
CCCL_SOURCE_DIR="${_cccl_root}"
CLANG_RESOURCE_DIR="${llvm_project_BINARY_DIR}/lib/clang/${LLVM_VERSION_MAJOR}"
CLANG_HEADERS_DIR="${llvm_project_SOURCE_DIR}/clang/lib/Headers"
HOSTJIT_INCLUDE_DIR="${CMAKE_CURRENT_SOURCE_DIR}/include"
)
if (CUDAToolkit_FOUND)
target_include_directories(
cccl.c.parallel.v2.hostjit_lib
PUBLIC ${CUDAToolkit_INCLUDE_DIRS}
)
cmake_path(GET CUDAToolkit_BIN_DIR PARENT_PATH CUDA_TOOLKIT_ROOT_FROM_CMAKE)
target_compile_definitions(
cccl.c.parallel.v2.hostjit_lib
PRIVATE
CUDA_TOOLKIT_PATH="${CUDA_TOOLKIT_ROOT_FROM_CMAKE}"
CUDA_SDK_VERSION="${CUDAToolkit_VERSION_MAJOR}.0"
)
endif()
# Link against LLVM/Clang/LLD
target_link_libraries(
cccl.c.parallel.v2.hostjit_lib
PUBLIC
# LLVM
LLVMCore
LLVMSupport
LLVMIRReader
LLVMMC
LLVMObject
LLVMX86CodeGen
LLVMX86AsmParser
LLVMX86Desc
LLVMX86Info
LLVMNVPTXCodeGen
LLVMNVPTXDesc
LLVMNVPTXInfo
LLVMLinker
LLVMPasses
# Clang
clangAST
clangBasic
clangCodeGen
clangDriver
clangFrontend
clangFrontendTool
clangLex
clangParse
clangSema
clangEdit
clangAnalysis
clangRewrite
clangSerialization
# LLD
$<IF:$<PLATFORM_ID:Windows>,lldCOFF,lldELF>
lldCommon
)
if (NOT WIN32)
target_link_libraries(cccl.c.parallel.v2.hostjit_lib PUBLIC dl)
endif()
if (CUDAToolkit_FOUND)
target_link_libraries(
cccl.c.parallel.v2.hostjit_lib
PUBLIC CUDA::cuda_driver CUDA::cudart
)
if (WIN32)
# On Windows, static CUDA libs are built with /MT which conflicts with
# the project's dynamic CRT (/MD). Use dynamic variants instead.
target_link_libraries(
cccl.c.parallel.v2.hostjit_lib
PUBLIC CUDA::nvJitLink CUDA::nvfatbin
)
else()
# Prefer static CUDA libs on Linux for self-contained binaries. If the
# toolchain (e.g. lite/pip CUDA installs or some Docker images) only ships
# the dynamic variants, fall back to those rather than failing configure.
foreach (_cudalib nvJitLink nvptxcompiler nvfatbin)
if (TARGET "CUDA::${_cudalib}_static")
target_link_libraries(
cccl.c.parallel.v2.hostjit_lib
PUBLIC "CUDA::${_cudalib}_static"
)
elseif (TARGET "CUDA::${_cudalib}")
target_link_libraries(
cccl.c.parallel.v2.hostjit_lib
PUBLIC "CUDA::${_cudalib}"
)
else()
message(
FATAL_ERROR
"hostjit needs CUDA::${_cudalib}[_static] but neither variant was "
"found by FindCUDAToolkit. Install the full CUDA toolkit "
"(libnvjitlink-dev / libnvfatbin-dev or equivalent)."
)
endif()
endforeach()
endif()
endif()
if (NOT MSVC)
target_compile_options(cccl.c.parallel.v2.hostjit_lib PRIVATE -fno-rtti)
endif()
set_target_properties(
cccl.c.parallel.v2.hostjit_lib
PROPERTIES CXX_STANDARD 20 POSITION_INDEPENDENT_CODE ON
)
# --------------------------------------------------------------------------
# Install clang headers into wheel (for self-sufficient packaging)
# --------------------------------------------------------------------------
# Clang CUDA headers we still use from the LLVM source tree.
# We DON'T install device_functions, math, or libdevice_declares — our local
# copies in cuda_minimal/ replace them.
set(
_clang_cuda_headers_needed
"${llvm_project_SOURCE_DIR}/clang/lib/Headers/__clang_cuda_math_forward_declares.h"
"${llvm_project_SOURCE_DIR}/clang/lib/Headers/__clang_cuda_builtin_vars.h"
"${llvm_project_SOURCE_DIR}/clang/lib/Headers/__clang_cuda_cmath.h"
"${llvm_project_SOURCE_DIR}/clang/lib/Headers/__clang_cuda_intrinsics.h"
"${llvm_project_SOURCE_DIR}/clang/lib/Headers/__clang_cuda_complex_builtins.h"
"${llvm_project_SOURCE_DIR}/clang/lib/Headers/__clang_cuda_texture_intrinsics.h"
)
install(
FILES ${_clang_cuda_headers_needed}
DESTINATION "cuda/cccl/headers/clang"
)
# Clang builtin C headers needed by our stubs and CUDA toolkit headers.
file(
GLOB _clang_stddef_headers
"${llvm_project_SOURCE_DIR}/clang/lib/Headers/__stddef_*.h"
)
set(
_clang_c_headers
"${llvm_project_SOURCE_DIR}/clang/lib/Headers/limits.h"
"${llvm_project_SOURCE_DIR}/clang/lib/Headers/stddef.h"
"${llvm_project_SOURCE_DIR}/clang/lib/Headers/stdint.h"
"${llvm_project_SOURCE_DIR}/clang/lib/Headers/__stddef_header_macro.h"
"${llvm_project_SOURCE_DIR}/clang/lib/Headers/float.h"
"${llvm_project_SOURCE_DIR}/clang/lib/Headers/__float_header_macro.h"
"${llvm_project_SOURCE_DIR}/clang/lib/Headers/inttypes.h"
${_clang_stddef_headers}
)
install(FILES ${_clang_c_headers} DESTINATION "cuda/cccl/headers/clang")
# Hostjit's minimal CUDA runtime headers (replacements for upstream clang headers)
set(
_hostjit_cuda_minimal_dir
"${CMAKE_CURRENT_SOURCE_DIR}/include/hostjit/cuda_minimal"
)
file(GLOB _hostjit_cuda_minimal_headers "${_hostjit_cuda_minimal_dir}/*.h")
install(
FILES ${_hostjit_cuda_minimal_headers}
DESTINATION "cuda/cccl/headers/hostjit/cuda_minimal"
)
# Hostjit's stub headers (minimal C++ standard library stubs for device compilation)
# Use GLOB_RECURSE + DIRECTORY so subdirectory overrides (e.g. cuda/std/__cstdlib/)
# are also installed alongside the top-level stubs.
install(
DIRECTORY "${_hostjit_cuda_minimal_dir}/stubs/"
DESTINATION "cuda/cccl/headers/hostjit/cuda_minimal/stubs"
)
# On Windows with multi-config generators (Visual Studio), exclude hostjit
# targets from Debug builds — the LLVM Debug build causes stack overflows.
if (MSVC)
set_target_properties(
cccl.c.parallel.v2.hostjit_lib
PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_DEBUG TRUE
)
endif()

237
cccl_upstream/c/parallel.v2/src/hostjit/codegen/bitcode.cpp Normal file
View File

@@ -0,0 +1,237 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <functional>
#include <stdexcept>
#include <string_view>
#include <hostjit/codegen/bitcode.hpp>
#include <hostjit/compiler.hpp>
namespace hostjit::codegen
{
namespace
{
bool write_file(const char* data, size_t size, const std::string& path)
{
std::ofstream f(path, std::ios::binary);
if (!f)
{
return false;
}
f.write(data, static_cast<std::streamsize>(size));
return f.good();
}
std::string make_temp_path(const std::string& prefix, uintptr_t id, const std::string& ext)
{
return (std::filesystem::temp_directory_path() / (prefix + std::to_string(id) + ext)).string();
}
} // anonymous namespace
BitcodeCollector::BitcodeCollector(CompilerConfig& config, uintptr_t unique_id)
: config_(config)
, unique_id_(unique_id)
{}
bool BitcodeCollector::is_bitcode_op(cccl_op_t op)
{
return (op.code_type == CCCL_OP_LLVM_IR || op.code_type == CCCL_OP_LTOIR) && op.code != nullptr && op.code_size > 0;
}
void BitcodeCollector::add_raw_bitcode(const char* data, size_t size, const std::string& name)
{
if (!data || size == 0)
{
return;
}
// Dedup by content hash: identical bitcode bytes define identical symbols
// (e.g. two PointerIterator<int>s sharing the same advance LTOIR). Adding
// both would make nvJitLink fail with "symbol multiply defined".
const auto hash = std::hash<std::string_view>{}(std::string_view(data, size));
if (!added_content_hashes_.insert(hash).second)
{
return; // exact same bytes already added
}
// Magic-byte routing: LLVM bitcode starts with "BC" (0x42 0x43) and goes to
// LLVM's bitcode linker so it can be inlined into the CUB module at the IR
// level. Anything else is treated as LTO-IR (binary fatbin container) and
// fed to nvJitLink. CPP_SOURCE never reaches here: main ops are dispatched
// by code_type in add_op_code, and per-extra C++ source is dispatched by
// extra_code_types[i] in the extras loop below — both call compile_and_add
// directly.
const bool is_llvm_bitcode =
size >= 2 && static_cast<unsigned char>(data[0]) == 0x42 && static_cast<unsigned char>(data[1]) == 0x43;
const char* ext = is_llvm_bitcode ? ".bc" : ".ltoir";
auto path = make_temp_path("cccl_" + name + "_", unique_id_, ext);
if (!write_file(data, size, path))
{
return;
}
if (is_llvm_bitcode)
{
config_.device_bitcode_files.push_back(path);
}
else
{
config_.device_ltoir_files.push_back(path);
}
temp_paths_.push_back(path);
}
bool BitcodeCollector::compile_and_add(const char* source, size_t source_size, const std::string& name)
{
// Dedup by source-content hash: two PointerIterator<int> children in the
// same zip produce identical CPP source that defines the same symbol; without
// this guard the LLVM linker fails with "symbol multiply defined".
const auto hash = std::hash<std::string_view>{}(std::string_view(source, source_size));
if (!added_content_hashes_.insert(hash).second)
{
return true;
}
hostjit::CUDACompiler compiler;
std::string src(source, source_size);
auto result = compiler.compileToDeviceBitcode(src, config_);
if (!result.success)
{
fprintf(stderr, "\nERROR compiling %s to bitcode: %s\n", name.c_str(), result.diagnostics.c_str());
return false;
}
auto path = make_temp_path("cccl_" + name + "_", unique_id_, ".bc");
if (write_file(result.bitcode.data(), result.bitcode.size(), path))
{
config_.device_bitcode_files.push_back(path);
temp_paths_.push_back(path);
return true;
}
return false;
}
void BitcodeCollector::add_op_code(cccl_op_t& op, const std::string& name)
{
if (!op.code || op.code_size == 0)
{
return;
}
// Deduplicate: if two iterators share the same symbol (e.g. two CountingIterators
// of the same type), only compile/link the bitcode once.
if (op.name && op.name[0])
{
if (!added_symbols_.insert(std::string(op.name)).second)
{
return; // already added
}
}
if (op.code_type == CCCL_OP_CPP_SOURCE)
{
compile_and_add(op.code, op.code_size, name);
}
else
{
add_raw_bitcode(op.code, op.code_size, name);
}
// Also link any extra modules (child iterator ops, numba-compiled ops).
int extra_counter = 0;
if (op.num_extra_ltoirs > 0 && (!op.extra_ltoirs || !op.extra_ltoir_sizes))
{
throw std::runtime_error("cccl_op_t: extra_ltoirs and extra_ltoir_sizes must be non-null when num_extra_ltoirs > "
"0");
}
for (size_t i = 0; i < op.num_extra_ltoirs; ++i)
{
if (op.extra_ltoirs[i] && op.extra_ltoir_sizes[i] > 0)
{
auto extra_name = name + "_extra" + std::to_string(extra_counter++);
const auto* data = op.extra_ltoirs[i];
const auto data_sz = op.extra_ltoir_sizes[i];
if (!op.extra_code_types)
{
throw std::runtime_error("cccl_op_t: extra_code_types must be non-null when num_extra_ltoirs > 0");
}
const cccl_op_code_type t = op.extra_code_types[i];
if (t == CCCL_OP_CPP_SOURCE)
{
compile_and_add(data, data_sz, extra_name);
}
else
{
add_raw_bitcode(data, data_sz, extra_name);
}
}
}
}
void BitcodeCollector::add_op(cccl_op_t op, const std::string& label)
{
// Only add bitcode for LTOIR/LLVM_IR ops (CPP_SOURCE is embedded inline in the generated source)
if (is_bitcode_op(op))
{
add_raw_bitcode(op.code, op.code_size, label);
}
// Always process extras with per-entry dispatch.
int extra_counter = 0;
if (op.num_extra_ltoirs > 0 && (!op.extra_ltoirs || !op.extra_ltoir_sizes))
{
throw std::runtime_error("cccl_op_t: extra_ltoirs and extra_ltoir_sizes must be non-null when num_extra_ltoirs > "
"0");
}
for (size_t i = 0; i < op.num_extra_ltoirs; ++i)
{
if (op.extra_ltoirs[i] && op.extra_ltoir_sizes[i] > 0)
{
auto extra_name = label + "_extra" + std::to_string(extra_counter++);
const auto* data = op.extra_ltoirs[i];
const auto data_sz = op.extra_ltoir_sizes[i];
if (!op.extra_code_types)
{
throw std::runtime_error("cccl_op_t: extra_code_types must be non-null when num_extra_ltoirs > 0");
}
const cccl_op_code_type t = op.extra_code_types[i];
if (t == CCCL_OP_CPP_SOURCE)
{
compile_and_add(data, data_sz, extra_name);
}
else
{
add_raw_bitcode(data, data_sz, extra_name);
}
}
}
}
void BitcodeCollector::add_iterator(cccl_iterator_t it, const std::string& label_prefix)
{
if (it.type != CCCL_ITERATOR)
{
return;
}
add_op_code(it.advance, label_prefix + "_adv");
add_op_code(it.dereference, label_prefix + "_deref");
}
void BitcodeCollector::cleanup()
{
for (const auto& p : temp_paths_)
{
std::filesystem::remove(p);
}
temp_paths_.clear();
}
} // namespace hostjit::codegen

812
cccl_upstream/c/parallel.v2/src/hostjit/codegen/cub_call.cpp Normal file
View File

@@ -0,0 +1,812 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <format>
#include <fstream>
#include <map>
#include <memory>
#include <stdexcept>
#include <hostjit/codegen/bitcode.hpp>
#include <hostjit/codegen/cub_call.hpp>
#include <hostjit/codegen/iterators.hpp>
#include <hostjit/codegen/operators.hpp>
#include <hostjit/codegen/types.hpp>
namespace hostjit::codegen
{
CubCall CubCall::from(const char* include_header)
{
CubCall c;
c.include_ = include_header;
return c;
}
CubCall& CubCall::run(const char* cub_function)
{
cub_function_ = cub_function;
return *this;
}
CubCall& CubCall::name(const char* export_name)
{
fn_name_ = export_name;
return *this;
}
// Helper to find the accumulator type from the argument list.
// Priority: first cccl_value_t, then first input_t's value_type.
namespace
{
cccl_type_info find_accum_type(const std::vector<Arg>& args)
{
// Highest priority: explicit override
for (const auto& arg : args)
{
if (auto* fa = std::get_if<force_accum_type_t>(&arg))
{
return fa->type;
}
}
// First: look for cccl_value_t (init value defines accum type)
for (const auto& arg : args)
{
if (auto* val = std::get_if<cccl_value_t>(&arg))
{
return val->type;
}
}
// Second: future_val_t carries explicit type info
for (const auto& arg : args)
{
if (auto* fv = std::get_if<future_val_t>(&arg))
{
return fv->type;
}
}
// Fallback: first input iterator's value_type
for (const auto& arg : args)
{
if (auto* inp = std::get_if<input_t>(&arg))
{
return inp->it.value_type;
}
}
// Last resort: first output iterator
for (const auto& arg : args)
{
if (auto* outp = std::get_if<output_t>(&arg))
{
return outp->it.value_type;
}
}
return cccl_type_info{sizeof(int), alignof(int), CCCL_INT32};
}
} // anonymous namespace
namespace
{
// Returns true if `args` contains an env_stream_t — used to decide whether the
// shared-includes block needs to pull in <cuda/std/__execution/env.h>.
bool needs_env_include(const std::vector<Arg>& args)
{
for (const auto& arg : args)
{
if (std::holds_alternative<env_stream_t>(arg))
{
return true;
}
}
return false;
}
// Emits the system #includes + the CUB header. Hoisted from source() so
// multi-function compiles can emit this once and wrap N function bodies in N
// namespaces below.
std::string shared_includes(const std::string& cub_include, bool needs_tuple, bool needs_env)
{
std::string src = R"(#include <cuda_runtime.h>
#include <cuda_fp16.h>
#include <cuda/std/iterator>
#include <cuda/std/functional>
#include <cuda/functional>
)";
if (needs_tuple)
{
src += "#include <cuda/std/tuple>\n";
}
if (needs_env)
{
// Use the narrow internal env.h header rather than <cuda/std/execution>
// — the umbrella header pulls in pstl machinery that depends on <vector>
// and exception types not available in the hostjit environment.
src += "#include <cuda/std/__execution/env.h>\n";
src += "#include <cuda/stream_ref>\n";
}
src += std::format("#include <{}>\n\n", cub_include);
return src;
}
} // namespace
std::string CubCall::source() const
{
// Single-function source = shared includes + this CubCall's body.
return shared_includes(include_, tuple_inputs_, needs_env_include(args_)) + body();
}
std::string CubCall::body() const
{
// Pass 1: determine accumulator type
cccl_type_info accum_info = find_accum_type(args_);
std::string accum_preamble;
std::string accum_type = resolve_type(accum_info, "storage_t", accum_preamble);
// Counters for unique naming
int in_count = 0;
int out_count = 0;
int op_count = 0;
int val_count = 0;
// Accumulated sections
std::string preamble;
std::vector<std::string> params;
std::vector<std::string> setup_lines;
std::vector<std::string> cub_args;
// Lines emitted after the cub::DeviceX::Y(...) call and before the return —
// populated by post-call tags (e.g. selector_out_t capturing a DoubleBuffer's
// selector member).
std::vector<std::string> post_call_lines;
// Emit accum type
if (!accum_preamble.empty())
{
preamble += accum_preamble;
}
preamble += std::format("using accum_t = {};\n\n", accum_type);
// Shared alias cache: (size, alignment) → type name.
// Multiple iterators with the same unknown struct layout must share a single C++
// type so that CUB can move data between them (e.g. merge sort block loads).
std::map<std::pair<size_t, size_t>, std::string> struct_type_map;
int struct_type_counter = 0;
// Return a stable C++ element-type name for an iterator's value_type:
// - Known C type → C++ keyword (e.g. "int", "float")
// - Struct matching accum_t → "accum_t" (preserves operator compatibility)
// - Other struct → shared alias for this (size, alignment) layout
// Built-in C type sizes (CCCL_TYPE_ENUM → bytes). Used to detect a
// mismatch where the caller reports a primitive `vt.type` but `vt.size`
// says the element is wider — common when a custom struct happens to
// share the primitive's tag. In that case fall through to a storage
// struct so the iterator strides correctly.
auto builtin_size = [](cccl_type_enum t) -> size_t {
switch (t)
{
case CCCL_INT8:
case CCCL_UINT8:
case CCCL_BOOLEAN:
return 1;
case CCCL_INT16:
case CCCL_UINT16:
case CCCL_FLOAT16:
return 2;
case CCCL_INT32:
case CCCL_UINT32:
case CCCL_FLOAT32:
return 4;
case CCCL_INT64:
case CCCL_UINT64:
case CCCL_FLOAT64:
return 8;
default:
return 0;
}
};
auto iter_elem_type_name = [&](const cccl_type_info& vt) -> std::string {
auto name = get_type_name(vt.type);
if (!name.empty() && vt.size == builtin_size(vt.type))
{
return name;
}
if (vt.size == accum_info.size && vt.alignment == accum_info.alignment && vt.type == accum_info.type)
{
return "accum_t";
}
auto key = std::make_pair(vt.size, vt.alignment);
auto it = struct_type_map.find(key);
if (it != struct_type_map.end())
{
return it->second;
}
auto alias = std::format("__cccl_struct_{}_t", struct_type_counter++);
preamble += make_storage_type(alias.c_str(), vt.size, vt.alignment);
struct_type_map[key] = alias;
return alias;
};
// Pass 2: process each argument
for (const auto& arg : args_)
{
std::visit(
[&](auto&& a) {
using T = std::decay_t<decltype(a)>;
if constexpr (std::is_same_v<T, temp_storage_t>)
{
params.push_back("void* d_temp_storage");
cub_args.push_back("d_temp_storage");
}
else if constexpr (std::is_same_v<T, temp_bytes_t>)
{
params.push_back("size_t* temp_storage_bytes");
cub_args.push_back("*temp_storage_bytes");
}
else if constexpr (std::is_same_v<T, num_items_t>)
{
params.push_back(std::format("unsigned long long {}", a.name));
cub_args.push_back(std::format("(unsigned long long){}", a.name));
}
else if constexpr (std::is_same_v<T, stream_t>)
{
params.push_back("void* stream");
cub_args.push_back("(cudaStream_t)stream");
}
else if constexpr (std::is_same_v<T, env_stream_t>)
{
params.push_back("void* stream");
cub_args.push_back("::cuda::std::execution::env{::cuda::stream_ref{(cudaStream_t)stream}}");
}
else if constexpr (std::is_same_v<T, input_t>)
{
auto idx = in_count++;
auto struct_name = std::format("in_{}_it_t", idx);
auto var_name = std::format("in_{}", idx);
auto param_name = std::format("d_in_{}", idx);
auto value_type = iter_elem_type_name(a.it.value_type);
auto code = make_input_iterator(a.it, value_type, "accum_t", struct_name, var_name, param_name);
preamble += code.preamble;
params.push_back(std::format("void* {}", param_name));
setup_lines.push_back(code.setup_code);
cub_args.push_back(var_name);
}
else if constexpr (std::is_same_v<T, output_t>)
{
auto idx = out_count++;
auto struct_name = std::format("out_{}_it_t", idx);
auto var_name = std::format("out_{}", idx);
auto param_name = std::format("d_out_{}", idx);
auto value_type = iter_elem_type_name(a.it.value_type);
auto code = make_output_iterator(a.it, "accum_t", struct_name, var_name, param_name, value_type);
preamble += code.preamble;
params.push_back(std::format("void* {}", param_name));
setup_lines.push_back(code.setup_code);
cub_args.push_back(var_name);
}
else if constexpr (std::is_same_v<T, cccl_op_t>)
{
auto idx = op_count++;
auto functor_name = std::format("Op_{}", idx);
auto var_name = std::format("op_{}", idx);
auto state_param = std::format("op_{}_state", idx);
bool has_bc = BitcodeCollector::is_bitcode_op(a);
auto code = make_binary_op(a, accum_type, functor_name, var_name, state_param, has_bc);
preamble += code.preamble;
// Always emit op_state param for ABI stability (unused for stateless ops)
params.push_back(std::format("void* {}", state_param));
setup_lines.push_back(code.setup_code);
cub_args.push_back(var_name);
}
else if constexpr (std::is_same_v<T, cmp_t>)
{
auto idx = op_count++;
auto functor_name = std::format("CmpOp_{}", idx);
auto var_name = std::format("cmp_{}", idx);
auto state_param = std::format("cmp_{}_state", idx);
bool has_bc = BitcodeCollector::is_bitcode_op(a.op);
auto code = make_comparison_op(a.op, accum_type, functor_name, var_name, state_param, has_bc);
preamble += code.preamble;
params.push_back(std::format("void* {}", state_param));
setup_lines.push_back(code.setup_code);
cub_args.push_back(var_name);
}
else if constexpr (std::is_same_v<T, for_each_op_t>)
{
auto idx = op_count++;
auto functor_name = std::format("ForEachOp_{}", idx);
auto var_name = std::format("op_{}", idx);
auto state_param = std::format("op_{}_state", idx);
bool has_bc = BitcodeCollector::is_bitcode_op(a.op);
// The element type is the first input iterator's value_type, which
// CubCall has already resolved via find_accum_type.
const std::string elem_type = iter_elem_type_name(accum_info);
auto code = make_for_each_op(a.op, elem_type, functor_name, var_name, state_param, has_bc);
preamble += code.preamble;
params.push_back(std::format("void* {}", state_param));
setup_lines.push_back(code.setup_code);
cub_args.push_back(var_name);
}
else if constexpr (std::is_same_v<T, double_buffer_t>)
{
// Emit two void* params (in/out buffer state pointers), construct a
// cub::DoubleBuffer<elem_t> local with the given var_name, and pass
// the buffer to the CUB call. iter_elem_type_name resolves the
// element type the same way input/output iterators do.
const std::string elem_type = iter_elem_type_name(a.in_it.value_type);
const std::string var_name = a.var_name;
const auto in_param = var_name + "_in_state";
const auto out_param = var_name + "_out_state";
params.push_back(std::format("void* {}", in_param));
params.push_back(std::format("void* {}", out_param));
setup_lines.push_back(std::format(
"cub::DoubleBuffer<{0}> {1}(static_cast<{0}*>({2}), static_cast<{0}*>({3}));",
elem_type,
var_name,
in_param,
out_param));
cub_args.push_back(var_name);
}
else if constexpr (std::is_same_v<T, selector_out_t>)
{
// Emit a void* selector_out param and capture <buffer>.selector after
// the CUB call. Paired with a double_buffer_t whose var_name matches.
params.push_back("void* selector_out");
post_call_lines.push_back(std::format("*static_cast<int*>(selector_out) = {}.selector;", a.buffer_var_name));
}
else if constexpr (std::is_same_v<T, unary_op_t>)
{
auto idx = op_count++;
auto functor_name = std::format("UnaryOp_{}", idx);
auto var_name = std::format("op_{}", idx);
auto state_param = std::format("op_{}_state", idx);
bool has_bc = BitcodeCollector::is_bitcode_op(a.op);
// For unknown types the iterators use accum_t as fallback; the unary
// op functor must use the same names so CUB can match the types.
// Reuse the iterator's element-type resolver so a primitive `vt.type`
// with a custom-sized `vt.size` falls back to the same storage alias
// the iterator uses, rather than naming the wider element "int".
std::string in_type = iter_elem_type_name(a.in_type);
std::string out_type = iter_elem_type_name(a.out_type);
auto code = make_unary_op(a.op, in_type, out_type, functor_name, var_name, state_param, has_bc);
preamble += code.preamble;
params.push_back(std::format("void* {}", state_param));
setup_lines.push_back(code.setup_code);
cub_args.push_back(var_name);
}
else if constexpr (std::is_same_v<T, force_accum_type_t>)
{
// No-op: only influences accum type resolution, generates no code.
}
else if constexpr (std::is_same_v<T, future_val_t>)
{
auto idx = val_count++;
auto var_name = std::format("future_{}", idx);
auto param_name = std::format("future_{}_param", idx);
// The caller passes a device pointer; we wrap it in FutureValue<accum_t>
// so CUB fetches the init value from device memory at scan time.
params.push_back(std::format("void* {}", param_name));
setup_lines.push_back(
std::format("cub::FutureValue<accum_t> {}(static_cast<accum_t*>({}));", var_name, param_name));
cub_args.push_back(var_name);
}
else if constexpr (std::is_same_v<T, cccl_value_t>)
{
auto idx = val_count++;
auto var_name = std::format("val_{}", idx);
auto param_name = std::format("val_{}_ptr", idx);
params.push_back(std::format("void* {}", param_name));
setup_lines.push_back(std::format(
"accum_t {};\n __builtin_memcpy(&{}, {}, sizeof(accum_t));", var_name, var_name, param_name));
cub_args.push_back(var_name);
}
else if constexpr (std::is_same_v<T, typed_scalar_t>)
{
// Caller passes a host pointer; we memcpy onto the stack as the
// requested C++ type before calling CUB. The C++ type comes from
// a.type (cccl_type_info), the wrapper parameter is named
// `<a.name>_ptr`, and the local that CUB sees is `<a.name>`.
const std::string cpp_type = resolve_type(a.type, a.name, preamble);
const auto param_name = std::string(a.name) + "_ptr";
params.push_back(std::format("void* {}", param_name));
setup_lines.push_back(
std::format("{0} {1};\n __builtin_memcpy(&{1}, {2}, sizeof({0}));", cpp_type, a.name, param_name));
cub_args.push_back(a.name);
}
},
arg);
}
// When tuple_inputs_ is set, replace the individual input cub_args with a
// single make_tuple(...) expression covering all of them.
if (tuple_inputs_ && in_count > 1)
{
// Collect the first in_count cub_args that correspond to input iterators.
// Inputs are emitted first among iterator args, so they occupy the leading
// cub_args entries (after temp_storage/temp_bytes if present).
// Reconstruct: find and replace the in_0..in_N-1 vars with make_tuple.
std::vector<std::string> input_vars;
std::vector<std::string> other_args;
for (const auto& a : cub_args)
{
// Input vars are named "in_0", "in_1", etc.
if (a.starts_with("in_") && a.size() >= 4 && std::isdigit(a[3]))
{
input_vars.push_back(a);
}
else
{
other_args.push_back(a);
}
}
std::string tuple_arg = "::cuda::std::make_tuple(";
for (size_t i = 0; i < input_vars.size(); ++i)
{
if (i)
{
tuple_arg += ", ";
}
tuple_arg += input_vars[i];
}
tuple_arg += ")";
// Rebuild cub_args: replace all in_* with the single tuple arg (at original position of in_0)
cub_args.clear();
cub_args.push_back(tuple_arg);
for (const auto& a : other_args)
{
cub_args.push_back(a);
}
}
// Assemble the per-function body: preamble + extern "C" function defn.
// System #includes live in shared_includes(), emitted once at TU scope by
// either source() (single-fn) or compile() (multi-fn).
std::string src = preamble;
// Function signature
src += std::format("extern \"C\" _CCCL_VISIBILITY_EXPORT int {}(\n", fn_name_);
for (size_t i = 0; i < params.size(); ++i)
{
src += " " + params[i];
if (i + 1 < params.size())
{
src += ",\n";
}
}
src += ")\n{\n";
// Setup code
for (const auto& line : setup_lines)
{
src += " " + line + "\n";
}
src += "\n";
// CUB call
src += std::format(" cudaError_t err = {}(\n", cub_function_);
for (size_t i = 0; i < cub_args.size(); ++i)
{
src += " " + cub_args[i];
if (i + 1 < cub_args.size())
{
src += ",\n";
}
}
src += ");\n\n";
// Post-call lines (e.g., capturing a DoubleBuffer's selector).
for (const auto& line : post_call_lines)
{
src += " " + line + "\n";
}
if (!post_call_lines.empty())
{
src += "\n";
}
// Error return
src += R"( return (int)err;
}
)";
return src;
}
hostjit::CompilerConfig CubCall::make_jit_config(
int cc_major,
int cc_minor,
cccl_build_config* config,
const char* ctk_path,
const char* cccl_include_path,
const std::string& entry_point_name)
{
auto jit_config = hostjit::detectDefaultConfig();
jit_config.sm_version = cc_major * 10 + cc_minor;
jit_config.verbose = false;
jit_config.entry_point_name = entry_point_name;
if (ctk_path && ctk_path[0] != '\0')
{
jit_config.cuda_toolkit_path = ctk_path;
// Rebuild library_paths from the new toolkit root so the linker
// can find libcudart.so in the pip-installed layout.
jit_config.library_paths.clear();
for (const char* subdir : {"lib64", "lib"})
{
auto candidate = std::filesystem::path(ctk_path) / subdir;
if (std::filesystem::exists(candidate))
{
jit_config.library_paths.push_back(candidate.string());
}
}
}
if (cccl_include_path && cccl_include_path[0] != '\0')
{
jit_config.cccl_include_path = cccl_include_path;
// When CCCL headers are pip-installed, the hostjit cuda_minimal headers
// are installed alongside them under the parent directory:
// cccl_include_path = .../cuda/cccl/headers/include/
// hostjit headers = .../cuda/cccl/headers/hostjit/cuda_minimal/
// So derive hostjit_include_path as the parent of cccl_include_path.
if (jit_config.hostjit_include_path.empty()
|| !std::filesystem::exists(jit_config.hostjit_include_path + "/hostjit/cuda_minimal"))
{
auto parent = std::filesystem::path(cccl_include_path).parent_path().string();
if (std::filesystem::exists(parent + "/hostjit/cuda_minimal"))
{
jit_config.hostjit_include_path = parent;
}
}
}
if (config)
{
for (size_t i = 0; i < config->num_extra_include_dirs; ++i)
{
jit_config.include_paths.push_back(config->extra_include_dirs[i]);
}
for (size_t i = 0; i < config->num_extra_compile_flags; ++i)
{
std::string_view flag = config->extra_compile_flags[i];
if (flag.starts_with("-D"))
{
flag.remove_prefix(2);
if (auto eq = flag.find('='); eq != std::string_view::npos)
{
jit_config.macro_definitions[std::string{flag.substr(0, eq)}] = std::string{flag.substr(eq + 1)};
}
else
{
jit_config.macro_definitions[std::string{flag}] = "";
}
}
}
jit_config.enable_pch = config->enable_pch != 0;
jit_config.verbose = config->verbose != 0;
}
return jit_config;
}
CubCallResult CubCall::compile(
int cc_major, int cc_minor, cccl_build_config* config, const char* ctk_path, const char* cccl_include_path) const
{
// 1. Configure compiler
auto jit_config = make_jit_config(cc_major, cc_minor, config, ctk_path, cccl_include_path, fn_name_);
// 2. Auto-collect bitcode from ops and iterators
uintptr_t unique_id = reinterpret_cast<uintptr_t>(this);
BitcodeCollector bitcode(jit_config, unique_id);
int op_idx = 0;
int in_idx = 0;
int out_idx = 0;
collect_bitcode(bitcode, op_idx, in_idx, out_idx);
// 3. Generate source
std::string cuda_source = source();
if (const char* dump_path = std::getenv("CUBCALL_DUMP_SOURCE"))
{
std::ofstream f(dump_path);
f << cuda_source;
}
// 4. Compile. unique_ptr ensures the JITCompiler is freed if the next two
// checks throw; .release() transfers ownership to CubCallResult on success.
auto compiler = std::make_unique<JITCompiler>(jit_config);
if (!compiler->compile(cuda_source))
{
std::string err = compiler->getLastError();
bitcode.cleanup();
throw std::runtime_error("CubCall compilation failed: " + err);
}
bitcode.cleanup();
// 5. Extract function pointer
using fn_t = int (*)(void*, ...);
auto fn = compiler->getFunction<fn_t>(fn_name_);
if (!fn)
{
throw std::runtime_error("CubCall function lookup failed: " + compiler->getLastError());
}
// 6. Copy cubin
auto cubin = compiler->getCubin();
return CubCallResult{compiler.release(), reinterpret_cast<void*>(fn), std::move(cubin)};
}
void CubCall::collect_bitcode(BitcodeCollector& bitcode, int& op_idx, int& in_idx, int& out_idx) const
{
for (const auto& arg : args_)
{
std::visit(
[&](auto&& a) {
using T = std::decay_t<decltype(a)>;
if constexpr (std::is_same_v<T, cccl_op_t>)
{
bitcode.add_op(a, std::format("op_{}", op_idx++));
}
else if constexpr (std::is_same_v<T, cmp_t>)
{
bitcode.add_op(a.op, std::format("cmp_{}", op_idx++));
}
else if constexpr (std::is_same_v<T, unary_op_t>)
{
bitcode.add_op(a.op, std::format("op_{}", op_idx++));
}
else if constexpr (std::is_same_v<T, for_each_op_t>)
{
bitcode.add_op(a.op, std::format("op_{}", op_idx++));
}
else if constexpr (std::is_same_v<T, input_t>)
{
bitcode.add_iterator(a.it, std::format("in_{}", in_idx++));
}
else if constexpr (std::is_same_v<T, output_t>)
{
bitcode.add_iterator(a.it, std::format("out_{}", out_idx++));
}
},
arg);
}
}
MultiCubCallResult CubCall::compile(
std::initializer_list<CubCall> calls,
int cc_major,
int cc_minor,
cccl_build_config* config,
const char* ctk_path,
const char* cccl_include_path)
{
if (calls.size() == 0)
{
throw std::runtime_error("CubCall::compile: empty CubCall list");
}
// All CubCalls must share the same CUB header — we emit it once at the top
// of the merged TU. (If a future use case needs heterogeneous includes,
// extend this to union the set; for now keep it strict so silent mismatches
// can't slip through.)
const std::string& shared_include = calls.begin()->include_;
for (const auto& cb : calls)
{
if (cb.include_ != shared_include)
{
throw std::runtime_error("CubCall::compile: all CubCalls in a multi-compile must share the same .from(include) "
"header");
}
}
// Detect whether any CubCall needs the env / tuple system includes.
bool any_tuple = false;
bool any_env = false;
for (const auto& cb : calls)
{
any_tuple = any_tuple || cb.tuple_inputs_;
any_env = any_env || needs_env_include(cb.args_);
}
// entry_point_name is used to mark a single function as preserved during
// internalization. Use the first CubCall's name as the primary entry; the
// others will still be exported via extern "C" _CCCL_VISIBILITY_EXPORT so dlsym finds them.
auto jit_config = make_jit_config(cc_major, cc_minor, config, ctk_path, cccl_include_path, calls.begin()->fn_name_);
// Shared BitcodeCollector across all CubCalls — identical user-op or
// iterator bitcode referenced from multiple wrappers gets deduplicated by
// content hash + symbol name inside the collector.
uintptr_t unique_id = reinterpret_cast<uintptr_t>(&*calls.begin());
BitcodeCollector bitcode(jit_config, unique_id);
int op_idx = 0;
int in_idx = 0;
int out_idx = 0;
for (const auto& cb : calls)
{
cb.collect_bitcode(bitcode, op_idx, in_idx, out_idx);
}
// Build the merged source: shared includes at TU scope, then one
// `namespace fn_<i> { ... body() }` per CubCall.
// The extern "C" _CCCL_VISIBILITY_EXPORT symbols defined inside each
// namespace export under the global C-linkage name (no mangling),
// so dlsym(handle, cb.fn_name_) finds them.
std::string cuda_source = shared_includes(shared_include, any_tuple, any_env);
int i = 0;
for (const auto& cb : calls)
{
cuda_source += std::format("namespace fn_{} {{\n", i);
cuda_source += cb.body();
cuda_source += std::format("}} // namespace fn_{}\n\n", i);
++i;
}
if (const char* dump_path = std::getenv("CUBCALL_DUMP_SOURCE"))
{
std::ofstream f(dump_path);
f << cuda_source;
}
if (std::getenv("CUBCALL_PRINT_SOURCE"))
{
std::fprintf(stderr,
"\n===== CubCall merged JIT source [%zu fns] =====\n%s\n===== end =====\n",
calls.size(),
cuda_source.c_str());
}
// Single Clang compile for the whole TU.
auto compiler = std::make_unique<JITCompiler>(jit_config);
if (!compiler->compile(cuda_source))
{
std::string err = compiler->getLastError();
bitcode.cleanup();
throw std::runtime_error("CubCall::compile (multi) compilation failed: " + err);
}
bitcode.cleanup();
// dlsym each function by its export name (positional order matches input).
using fn_t = int (*)(void*, ...);
std::vector<void*> fn_ptrs;
fn_ptrs.reserve(calls.size());
for (const auto& cb : calls)
{
auto fn = compiler->getFunction<fn_t>(cb.fn_name_);
if (!fn)
{
throw std::runtime_error("CubCall::compile (multi) function lookup failed: " + cb.fn_name_);
}
fn_ptrs.push_back(reinterpret_cast<void*>(fn));
}
auto cubin = compiler->getCubin();
return MultiCubCallResult{compiler.release(), std::move(cubin), std::move(fn_ptrs)};
}
} // namespace hostjit::codegen

288
cccl_upstream/c/parallel.v2/src/hostjit/codegen/iterators.cpp Normal file
View File

@@ -0,0 +1,288 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#include <algorithm>
#include <cstddef>
#include <format>
#include <hostjit/codegen/iterators.hpp>
#include <hostjit/codegen/types.hpp>
namespace hostjit::codegen
{
namespace
{
// The iterator struct holds a `long long _delta` lazy-offset field, so its
// natural alignment is at least alignof(long long)==8. C++ rejects alignas
// values smaller than the natural alignment; clamp here so user iterators with
// small `it.alignment` (e.g. 1 for a `char` state) still produce a valid struct.
inline std::size_t struct_alignas(std::size_t it_alignment)
{
const std::size_t base = it_alignment > 0 ? it_alignment : 1;
return base < alignof(long long) ? alignof(long long) : base;
}
} // namespace
IteratorCode make_input_iterator(
cccl_iterator_t it,
const std::string& value_type_name,
const std::string& accum_type_name,
const std::string& struct_name,
const std::string& var_name,
const std::string& state_param)
{
IteratorCode result;
result.local_var = var_name;
if (it.type == CCCL_POINTER)
{
// For pointer iterators, the element type is value_type.
// When value_type_name is empty (unknown/struct type), resolve it from the iterator's
// value_type info to get a correctly-sized storage struct — falling back to accum_t
// would use the wrong element size if the value type differs from the accumulator.
std::string elem_type;
if (value_type_name.empty())
{
auto elem_alias = struct_name + "_elem_t";
elem_type = resolve_type(it.value_type, elem_alias.c_str(), result.preamble);
}
else
{
elem_type = value_type_name;
}
result.type_name = elem_type + "*";
result.preamble += std::format("using {} = {}*;\n\n", struct_name, elem_type);
result.setup_code = std::format("{} {} = static_cast<{}>({}); ", struct_name, var_name, struct_name, state_param);
}
else
{
// Custom iterator with state + advance + dereference
const std::string adv_name = (it.advance.name && it.advance.name[0]) ? it.advance.name : (var_name + "_advance");
const std::string deref_name =
(it.dereference.name && it.dereference.name[0]) ? it.dereference.name : (var_name + "_dereference");
auto input_val_type = value_type_name.empty() ? accum_type_name : value_type_name;
auto val_alias = var_name + "_value_t";
result.type_name = struct_name;
result.preamble = std::format("using {} = {};\n", val_alias, input_val_type);
result.preamble += std::format(
R"cpp(extern "C" __device__ void {}(void* state, const void* offset);
extern "C" __device__ void {}(const void* state, {}* result);
)cpp",
adv_name,
deref_name,
val_alias);
// Positional args: {0}=struct_name, {1}=val_alias, {2}=it.size, {3}=adv_name, {4}=deref_name, {5}=it.alignment
//
// Arithmetic ops (+, +=, ++) are __host__ __device__ so CUB's host
// dispatch (which does `iter += n` etc.) compiles in the freestanding
// host pass. They accumulate into `_delta` rather than calling the
// device-only `advance` bitcode. `operator*` (device-only) applies the
// accumulated `_delta` to a copy of state via `advance`, then derefs.
// `alignas({5})` matches the iterator's declared state alignment so the
// user-supplied advance/dereference (which casts state as a pointer/etc.)
// sees properly-aligned memory.
result.preamble += std::format(
R"cpp(struct alignas({5}) {0} {{
using value_type = {1};
using difference_type = long long;
using pointer = {1}*;
using reference = {1};
using iterator_category = cuda::std::random_access_iterator_tag;
alignas({5}) char state[{2}];
long long _delta = 0;
__host__ __device__ {0} operator+(difference_type n) const {{
{0} copy = *this;
copy._delta += n;
return copy;
}}
__host__ __device__ {0}& operator+=(difference_type n) {{
_delta += n;
return *this;
}}
__host__ __device__ {0}& operator++() {{ return *this += 1; }}
__host__ __device__ {0} operator++(int) {{ {0} tmp = *this; ++(*this); return tmp; }}
__host__ __device__ difference_type operator-(const {0}&) const {{ return 0; }}
__device__ {1} operator*() const {{
{0} copy = *this;
if (copy._delta != 0) {{
long long offset = copy._delta;
{3}(copy.state, &offset);
}}
{1} result;
{4}(copy.state, &result);
return result;
}}
__device__ {1} operator[](difference_type n) const {{ return *(*this + n); }}
__host__ __device__ bool operator==(const {0}&) const {{ return false; }}
__host__ __device__ bool operator!=(const {0}&) const {{ return true; }}
}};
)cpp",
struct_name, // {0}
val_alias, // {1}
it.size, // {2}
adv_name, // {3}
deref_name, // {4}
struct_alignas(it.alignment)); // {5}
result.setup_code = std::format(
R"cpp({} {};
__builtin_memcpy({}.state, {}, {});)cpp",
struct_name,
var_name,
var_name,
state_param,
it.size);
}
return result;
}
IteratorCode make_output_iterator(
cccl_iterator_t it,
const std::string& accum_type_name,
const std::string& struct_name,
const std::string& var_name,
const std::string& state_param,
const std::string& value_type_name)
{
IteratorCode result;
result.local_var = var_name;
// For custom iterators the element type comes from the dereference function so the
// accum_t fallback is fine; for pointer iterators we resolve the actual value_type
// below to get the correct element size.
const std::string elem_type = value_type_name.empty() ? accum_type_name : value_type_name;
if (it.type == CCCL_POINTER)
{
// When value_type_name is empty (unknown/struct type), resolve from the iterator's own
// value_type info so the element size is correct — not from accum_t which may differ.
std::string ptr_elem_type;
if (value_type_name.empty())
{
auto elem_alias = struct_name + "_elem_t";
ptr_elem_type = resolve_type(it.value_type, elem_alias.c_str(), result.preamble);
}
else
{
ptr_elem_type = value_type_name;
}
result.type_name = ptr_elem_type + "*";
result.preamble += std::format("using {} = {}*;\n\n", struct_name, ptr_elem_type);
result.setup_code = std::format("{} {} = static_cast<{}*>({});", struct_name, var_name, ptr_elem_type, state_param);
}
else
{
const std::string adv_name = (it.advance.name && it.advance.name[0]) ? it.advance.name : (var_name + "_advance");
const std::string deref_name =
(it.dereference.name && it.dereference.name[0]) ? it.dereference.name : (var_name + "_dereference");
auto proxy_name = var_name + "_proxy_t";
result.type_name = struct_name;
result.preamble = std::format(
R"cpp(extern "C" __device__ void {}(void* state, const void* offset);
extern "C" __device__ void {}(void* state, const void* value);
)cpp",
adv_name,
deref_name);
// The proxy carries a COPY of the iterator state, not a pointer to it.
// This is critical for indexed writes (output_it[i] = val): operator[] creates
// a temporary advanced iterator, calls operator* on it, and returns the proxy
// by value. After operator[] returns the temporary is destroyed, so a pointer
// to its state would be dangling. Storing the state bytes in the proxy itself
// makes the proxy self-contained and safe across that return.
// Proxy contains only `char state[N]` so its natural alignment is 1; the
// struct alignas is the bigger of the iterator's declared alignment and 1.
const std::size_t proxy_align = it.alignment > 0 ? it.alignment : 1;
result.preamble += std::format(
R"cpp(struct alignas({1}) {0} {{
alignas({1}) char state[{2}];
__device__ void operator=(const {3}& val) {{
{4}(state, &val);
}}
}};
)cpp",
proxy_name, // {0}
proxy_align, // {1}
it.size, // {2}
elem_type, // {3}
deref_name); // {4}
// Arithmetic ops (+, +=, ++) are __host__ __device__ so CUB's host
// dispatch compiles; they accumulate `_delta` instead of calling the
// device-only `advance` bitcode. operator* (device only) applies the
// accumulated `_delta` before constructing the proxy.
result.preamble += std::format(
R"cpp(struct alignas({5}) {0} {{
using value_type = {1};
using difference_type = long long;
using pointer = {1}*;
using reference = {2};
using iterator_category = cuda::std::random_access_iterator_tag;
alignas({5}) char state[{3}];
long long _delta = 0;
__host__ __device__ {0} operator+(difference_type n) const {{
{0} copy = *this;
copy._delta += n;
return copy;
}}
__host__ __device__ {0}& operator+=(difference_type n) {{
_delta += n;
return *this;
}}
__host__ __device__ {0}& operator++() {{ return *this += 1; }}
__host__ __device__ {0} operator++(int) {{ {0} tmp = *this; ++(*this); return tmp; }}
__host__ __device__ difference_type operator-(const {0}&) const {{ return 0; }}
__device__ reference operator*() const {{
{2} proxy;
__builtin_memcpy(proxy.state, state, {3});
if (_delta != 0) {{
long long offset = _delta;
{4}(proxy.state, &offset);
}}
return proxy;
}}
__device__ reference operator[](difference_type n) const {{ return *(*this + n); }}
}};
)cpp",
struct_name, // {0}
elem_type, // {1}
proxy_name, // {2}
it.size, // {3}
adv_name, // {4}
struct_alignas(it.alignment)); // {5}
result.setup_code = std::format(
R"cpp({} {};
__builtin_memcpy({}.state, {}, {});)cpp",
struct_name,
var_name,
var_name,
state_param,
it.size);
}
return result;
}
} // namespace hostjit::codegen

613
cccl_upstream/c/parallel.v2/src/hostjit/codegen/operators.cpp Normal file
View File

@@ -0,0 +1,613 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#include <format>
#include <hostjit/codegen/operators.hpp>
namespace hostjit::codegen
{
namespace
{
std::string generate_op_source(cccl_op_t op, bool has_bitcode, bool is_stateful)
{
const std::string op_name = (op.name && op.name[0]) ? op.name : "user_op";
std::string src;
if (op.code_type == CCCL_OP_CPP_SOURCE && op.code && op.code_size > 0)
{
// Embed C++ source directly
src += std::string(op.code, op.code_size) + "\n\n";
}
else if (has_bitcode)
{
// Extern declaration for bitcode-linked operation
if (is_stateful)
{
src += std::format("extern \"C\" __device__ void {}(void* state, void* a_ptr, void* b_ptr, void* out_ptr);\n\n",
op_name);
}
else
{
src += std::format("extern \"C\" __device__ void {}(void* a_ptr, void* b_ptr, void* out_ptr);\n\n", op_name);
}
}
return src;
}
std::string generate_binary_functor(cccl_op_t op, const std::string& accum_type, const std::string& functor_name)
{
const std::string op_name = (op.name && op.name[0]) ? op.name : "user_op";
const bool is_stateful = (op.type == CCCL_STATEFUL);
// Templated operator() lets CUB instantiate the functor with whatever
// element types its kernel deduces (important for binary transform with
// two differently-typed input iterators). The user's bitcode hop takes
// void* anyway, so the concrete arg types only need to be addressable.
if (is_stateful)
{
// Embed the user's state bytes inline. When CUB launches a kernel with
// this functor by value, the bytes ride along in the launch-arg buffer
// into device constant memory, so the address handed to the user's op
// (`state_bytes`) is a valid device-side pointer. Storing a host pointer
// here would crash on first device-side dereference.
const size_t state_size = op.size > 0 ? op.size : 1;
const size_t state_align = op.alignment > 0 ? op.alignment : 1;
return std::format(
R"cpp(struct {0} {{
alignas({3}) unsigned char state_bytes[{4}];
template <typename _A, typename _B>
__host__ __device__ __forceinline__
{1} operator()(const _A& a, const _B& b) const {{
{1} result;
{2}((void*)state_bytes, (void*)&a, (void*)&b, (void*)&result);
return result;
}}
}};
)cpp",
functor_name,
accum_type,
op_name,
state_align,
state_size);
}
else
{
return std::format(
R"cpp(struct {0} {{
template <typename _A, typename _B>
__host__ __device__ __forceinline__
{1} operator()(const _A& a, const _B& b) const {{
{1} result;
{2}((void*)&a, (void*)&b, (void*)&result);
return result;
}}
}};
)cpp",
functor_name,
accum_type,
op_name);
}
}
std::string generate_comparison_functor(cccl_op_t op, const std::string& key_type, const std::string& functor_name)
{
const std::string op_name = (op.name && op.name[0]) ? op.name : "user_op";
const bool is_stateful = (op.type == CCCL_STATEFUL);
if (is_stateful)
{
// See generate_binary_functor: state must travel by value via kernel-arg
// copy, not by host pointer, or the device-side deref crashes.
const size_t state_size = op.size > 0 ? op.size : 1;
const size_t state_align = op.alignment > 0 ? op.alignment : 1;
return std::format(
R"cpp(struct {0} {{
alignas({3}) unsigned char state_bytes[{4}];
__host__ __device__ __forceinline__
bool operator()(const {1}& a, const {2}& b) const {{
bool result;
{5}((void*)state_bytes, (void*)&a, (void*)&b, (void*)&result);
return result;
}}
}};
)cpp",
functor_name,
key_type,
key_type,
state_align,
state_size,
op_name);
}
else
{
return std::format(
R"cpp(struct {} {{
__host__ __device__ __forceinline__
bool operator()(const {}& a, const {}& b) const {{
bool result;
{}((void*)&a, (void*)&b, (void*)&result);
return result;
}}
}};
)cpp",
functor_name,
key_type,
key_type,
op_name);
}
}
// Returns the cuda::std (or cuda::) functor type string for a well-known binary op, or nullptr if not well-known.
const char* get_well_known_binary_functor_type(cccl_op_kind_t kind)
{
switch (kind)
{
case CCCL_PLUS:
return "::cuda::std::plus<>";
case CCCL_MINUS:
return "::cuda::std::minus<>";
case CCCL_MULTIPLIES:
return "::cuda::std::multiplies<>";
case CCCL_DIVIDES:
return "::cuda::std::divides<>";
case CCCL_MODULUS:
return "::cuda::std::modulus<>";
case CCCL_EQUAL_TO:
return "::cuda::std::equal_to<>";
case CCCL_NOT_EQUAL_TO:
return "::cuda::std::not_equal_to<>";
case CCCL_GREATER:
return "::cuda::std::greater<>";
case CCCL_LESS:
return "::cuda::std::less<>";
case CCCL_GREATER_EQUAL:
return "::cuda::std::greater_equal<>";
case CCCL_LESS_EQUAL:
return "::cuda::std::less_equal<>";
case CCCL_LOGICAL_AND:
return "::cuda::std::logical_and<>";
case CCCL_LOGICAL_OR:
return "::cuda::std::logical_or<>";
case CCCL_BIT_AND:
return "::cuda::std::bit_and<>";
case CCCL_BIT_OR:
return "::cuda::std::bit_or<>";
case CCCL_BIT_XOR:
return "::cuda::std::bit_xor<>";
case CCCL_MINIMUM:
return "::cuda::minimum<>";
case CCCL_MAXIMUM:
return "::cuda::maximum<>";
default:
return nullptr;
}
}
// Returns the cuda::std functor type string for a well-known unary op, or nullptr if not well-known.
const char* get_well_known_unary_functor_type(cccl_op_kind_t kind)
{
switch (kind)
{
case CCCL_LOGICAL_NOT:
return "::cuda::std::logical_not<>";
case CCCL_BIT_NOT:
return "::cuda::std::bit_not<>";
case CCCL_IDENTITY:
return "::cuda::std::identity";
case CCCL_NEGATE:
return "::cuda::std::negate<>";
default:
return nullptr;
}
}
// Returns the C++ operator symbol for a well-known op, or nullptr if none.
const char* get_well_known_op_symbol(cccl_op_kind_t kind)
{
switch (kind)
{
case CCCL_PLUS:
return "+";
case CCCL_MINUS:
return "-";
case CCCL_MULTIPLIES:
return "*";
case CCCL_DIVIDES:
return "/";
case CCCL_MODULUS:
return "%";
case CCCL_EQUAL_TO:
return "==";
case CCCL_NOT_EQUAL_TO:
return "!=";
case CCCL_GREATER:
return ">";
case CCCL_LESS:
return "<";
case CCCL_GREATER_EQUAL:
return ">=";
case CCCL_LESS_EQUAL:
return "<=";
case CCCL_LOGICAL_AND:
return "&&";
case CCCL_LOGICAL_OR:
return "||";
case CCCL_LOGICAL_NOT:
return "!";
case CCCL_BIT_AND:
return "&";
case CCCL_BIT_OR:
return "|";
case CCCL_BIT_XOR:
return "^";
case CCCL_BIT_NOT:
return "~";
case CCCL_NEGATE:
return "-";
default:
return nullptr;
}
}
// Generate preamble for a well-known binary op.
// For custom types with user-provided code, declares the extern "C" function
// and generates an operator overload that calls it.
// For primitive types without user code, no preamble is needed.
std::string
generate_well_known_preamble(cccl_op_t op, const std::string& accum_type, bool has_bitcode, bool is_comparison)
{
const std::string op_name = (op.name && op.name[0]) ? op.name : "user_op";
const std::string return_type = is_comparison ? "bool" : accum_type;
const char* symbol = get_well_known_op_symbol(op.type);
bool has_user_code = has_bitcode || (op.code_type == CCCL_OP_CPP_SOURCE && op.code && op.code_size > 0);
if (!has_user_code)
{
// Pure well-known op on a primitive type — no preamble needed.
return "";
}
std::string src;
if (op.code_type == CCCL_OP_CPP_SOURCE && op.code && op.code_size > 0)
{
// Embed C++ source directly (may contain type definitions).
src += std::string(op.code, op.code_size) + "\n\n";
}
// Declare the extern "C" function from bitcode.
if (has_bitcode)
{
src += std::format("extern \"C\" __device__ void {}(void* a_ptr, void* b_ptr, void* out_ptr);\n\n", op_name);
}
// Generate an operator overload that calls the user-provided function,
// so cuda::std::plus<> (etc.) can use it on custom types.
if (symbol)
{
src += std::format(
R"cpp(__device__ {0} operator{1}(const {2}& lhs, const {2}& rhs) {{
{0} ret;
{3}((void*)&lhs, (void*)&rhs, (void*)&ret);
return ret;
}}
)cpp",
return_type,
symbol,
accum_type,
op_name);
}
return src;
}
// Generate preamble for a well-known unary op with user-provided code.
// The operator overload lets the cuda::std functor invoke that code for a
// custom type. Primitive types without user code need no preamble.
std::string generate_well_known_unary_preamble(
cccl_op_t op, const std::string& in_type, const std::string& out_type, bool has_bitcode)
{
const std::string op_name = (op.name && op.name[0]) ? op.name : "user_op";
const char* symbol = get_well_known_op_symbol(op.type);
bool has_user_code = has_bitcode || (op.code_type == CCCL_OP_CPP_SOURCE && op.code && op.code_size > 0);
if (!has_user_code)
{
return "";
}
std::string src;
if (op.code_type == CCCL_OP_CPP_SOURCE && op.code && op.code_size > 0)
{
src += std::string(op.code, op.code_size) + "\n\n";
}
if (has_bitcode)
{
src += std::format("extern \"C\" __device__ void {}(void* a_ptr, void* out_ptr);\n\n", op_name);
}
if (symbol)
{
src += std::format(
R"cpp(__device__ {0} operator{1}(const {2}& value) {{
{0} ret;
{3}((void*)&value, (void*)&ret);
return ret;
}}
)cpp",
out_type,
symbol,
in_type,
op_name);
}
return src;
}
} // anonymous namespace
OperatorCode make_binary_op(
cccl_op_t op,
const std::string& accum_type,
const std::string& functor_name,
const std::string& var_name,
const std::string& state_param,
bool has_bitcode)
{
// For well-known operations, use cuda::std functors directly.
// For custom types, generate an operator overload that wraps the user-provided function.
// If the caller provided bitcode, prefer it: the well-known functor (e.g.
// cuda::std::plus<void>) may not be invocable on the custom value type.
const char* well_known_type = get_well_known_binary_functor_type(op.type);
if (well_known_type && !has_bitcode)
{
OperatorCode result;
result.local_var = var_name;
result.preamble = generate_well_known_preamble(op, accum_type, has_bitcode, /*is_comparison=*/false);
result.setup_code = std::format("{} {}{{}};", well_known_type, var_name);
return result;
}
const bool is_stateful = (op.type == CCCL_STATEFUL);
OperatorCode result;
result.local_var = var_name;
result.preamble = generate_op_source(op, has_bitcode, is_stateful);
result.preamble += generate_binary_functor(op, accum_type, functor_name);
if (is_stateful)
{
const size_t state_size = op.size > 0 ? op.size : 1;
result.setup_code = std::format(
"{0} {1}; __builtin_memcpy({1}.state_bytes, {2}, {3});", functor_name, var_name, state_param, state_size);
}
else
{
result.setup_code = std::format("{} {};", functor_name, var_name);
}
return result;
}
OperatorCode make_unary_op(
cccl_op_t op,
const std::string& in_type,
const std::string& out_type,
const std::string& functor_name,
const std::string& var_name,
const std::string& state_param,
bool has_bitcode)
{
// Well-known operations map directly to cuda::std unary functors. If the
// caller provided bitcode, prefer it because the functor may not be
// invocable on the user's custom value type.
const char* well_known_type = get_well_known_unary_functor_type(op.type);
if (well_known_type && !has_bitcode)
{
OperatorCode result;
result.local_var = var_name;
result.preamble = generate_well_known_unary_preamble(op, in_type, out_type, has_bitcode);
result.setup_code = std::format("{} {}{{}};", well_known_type, var_name);
return result;
}
const bool is_stateful = (op.type == CCCL_STATEFUL);
const std::string op_name = (op.name && op.name[0]) ? op.name : "user_op";
OperatorCode result;
result.local_var = var_name;
// Preamble: extern decl or embedded C++ source
if (op.code_type == CCCL_OP_CPP_SOURCE && op.code && op.code_size > 0)
{
result.preamble += std::string(op.code, op.code_size) + "\n\n";
}
else if (has_bitcode)
{
if (is_stateful)
{
result.preamble +=
std::format("extern \"C\" __device__ void {}(void* state, void* a_ptr, void* result_ptr);\n\n", op_name);
}
else
{
result.preamble += std::format("extern \"C\" __device__ void {}(void* a_ptr, void* result_ptr);\n\n", op_name);
}
}
// Functor struct
if (is_stateful)
{
// See generate_binary_functor: state must travel by value via kernel-arg
// copy, not by host pointer, or the device-side deref crashes.
const size_t state_size = op.size > 0 ? op.size : 1;
const size_t state_align = op.alignment > 0 ? op.alignment : 1;
result.preamble += std::format(
R"cpp(struct {0} {{
alignas({4}) unsigned char state_bytes[{5}];
__host__ __device__ __forceinline__
{1} operator()(const {2}& a) const {{
{3} result;
{6}((void*)state_bytes, (void*)&a, (void*)&result);
return result;
}}
}};
)cpp",
functor_name,
out_type,
in_type,
out_type,
state_align,
state_size,
op_name);
result.setup_code = std::format(
"{0} {1}; __builtin_memcpy({1}.state_bytes, {2}, {3});", functor_name, var_name, state_param, state_size);
}
else
{
result.preamble += std::format(
R"cpp(struct {} {{
__host__ __device__ __forceinline__
{} operator()(const {}& a) const {{
{} result;
{}((void*)&a, (void*)&result);
return result;
}}
}};
)cpp",
functor_name,
out_type,
in_type,
out_type,
op_name);
result.setup_code = std::format("{} {};", functor_name, var_name);
}
return result;
}
OperatorCode make_comparison_op(
cccl_op_t op,
const std::string& key_type,
const std::string& functor_name,
const std::string& var_name,
const std::string& state_param,
bool has_bitcode)
{
const char* well_known_type = get_well_known_binary_functor_type(op.type);
if (well_known_type && !has_bitcode)
{
OperatorCode result;
result.local_var = var_name;
result.preamble = generate_well_known_preamble(op, key_type, has_bitcode, /*is_comparison=*/true);
result.setup_code = std::format("{} {}{{}};", well_known_type, var_name);
return result;
}
const bool is_stateful = (op.type == CCCL_STATEFUL);
OperatorCode result;
result.local_var = var_name;
result.preamble = generate_op_source(op, has_bitcode, is_stateful);
result.preamble += generate_comparison_functor(op, key_type, functor_name);
if (is_stateful)
{
const size_t state_size = op.size > 0 ? op.size : 1;
result.setup_code = std::format(
"{0} {1}; __builtin_memcpy({1}.state_bytes, {2}, {3});", functor_name, var_name, state_param, state_size);
}
else
{
result.setup_code = std::format("{} {};", functor_name, var_name);
}
return result;
}
OperatorCode make_for_each_op(
cccl_op_t op,
const std::string& elem_type,
const std::string& functor_name,
const std::string& var_name,
const std::string& state_param,
bool has_bitcode)
{
const std::string op_name = (op.name && op.name[0]) ? op.name : "user_op";
const bool is_stateful = (op.type == CCCL_STATEFUL);
OperatorCode result;
result.local_var = var_name;
// Forward declaration / embedded source.
if (op.code_type == CCCL_OP_CPP_SOURCE && op.code && op.code_size > 0)
{
result.preamble += std::string(op.code, op.code_size) + "\n\n";
}
else if (has_bitcode)
{
if (is_stateful)
{
result.preamble +=
std::format("extern \"C\" __device__ void {}(void* state, {}* input);\n\n", op_name, elem_type);
}
else
{
result.preamble += std::format("extern \"C\" __device__ void {}({}* input);\n\n", op_name, elem_type);
}
}
if (is_stateful)
{
const size_t state_size = op.size > 0 ? op.size : 1;
const size_t state_align = op.alignment > 0 ? op.alignment : 1;
result.preamble += std::format(
R"cpp(struct {0} {{
alignas({3}) unsigned char state_bytes[{4}];
__device__ __forceinline__ void operator()({1}& elem) const {{ {2}((void*)state_bytes, &elem); }}
}};
)cpp",
functor_name,
elem_type,
op_name,
state_align,
state_size);
result.setup_code = std::format(
"{0} {1}; __builtin_memcpy({1}.state_bytes, {2}, {3});", functor_name, var_name, state_param, state_size);
}
else
{
result.preamble += std::format(
R"cpp(struct {0} {{
__device__ __forceinline__ void operator()({1}& elem) const {{ {2}(&elem); }}
}};
)cpp",
functor_name,
elem_type,
op_name);
result.setup_code = std::format("{} {}{{}};", functor_name, var_name);
}
return result;
}
} // namespace hostjit::codegen

73
cccl_upstream/c/parallel.v2/src/hostjit/codegen/types.cpp Normal file
View File

@@ -0,0 +1,73 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#include <format>
#include <hostjit/codegen/types.hpp>
namespace hostjit::codegen
{
std::string get_type_name(cccl_type_enum type)
{
switch (type)
{
case CCCL_INT8:
return "char";
case CCCL_INT16:
return "short";
case CCCL_INT32:
return "int";
case CCCL_INT64:
return "long long";
case CCCL_UINT8:
return "unsigned char";
case CCCL_UINT16:
return "unsigned short";
case CCCL_UINT32:
return "unsigned int";
case CCCL_UINT64:
return "unsigned long long";
case CCCL_FLOAT16:
return "__half";
case CCCL_FLOAT32:
return "float";
case CCCL_FLOAT64:
return "double";
case CCCL_BOOLEAN:
return "bool";
default:
return "";
}
}
std::string make_storage_type(const char* name, size_t size, size_t alignment)
{
return std::format(
R"cpp(struct __align__({}) {} {{
char data[{}];
}};
)cpp",
alignment,
name,
size);
}
std::string resolve_type(cccl_type_info info, const char* fallback_alias, std::string& out_preamble)
{
auto name = get_type_name(info.type);
if (!name.empty())
{
return name;
}
// Custom type: emit storage struct definition, return alias
out_preamble += make_storage_type(fallback_alias, info.size, info.alignment);
return fallback_alias;
}
} // namespace hostjit::codegen

1727
cccl_upstream/c/parallel.v2/src/hostjit/compiler.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

175
cccl_upstream/c/parallel.v2/src/hostjit/config.cpp Normal file
View File

@@ -0,0 +1,175 @@
#include <cstdlib>
#include <filesystem>
#include <iostream>
#include <cuda_runtime.h>
#include <hostjit/config.hpp>
namespace hostjit
{
CompilerConfig detectDefaultConfig()
{
CompilerConfig config;
// Detect CUDA toolkit path
if (const char* env = std::getenv("CUDA_PATH"))
{
config.cuda_toolkit_path = env;
}
else if (const char* env = std::getenv("CUDA_HOME"))
{
config.cuda_toolkit_path = env;
}
#ifdef CUDA_TOOLKIT_PATH
else
{
config.cuda_toolkit_path = CUDA_TOOLKIT_PATH;
}
#endif
// Set up library paths if CUDA toolkit was found
if (!config.cuda_toolkit_path.empty())
{
std::filesystem::path lib64_path = std::filesystem::path(config.cuda_toolkit_path) / "lib64";
std::filesystem::path lib_path = std::filesystem::path(config.cuda_toolkit_path) / "lib";
if (std::filesystem::exists(lib64_path))
{
config.library_paths.push_back(lib64_path.string());
}
else if (std::filesystem::exists(lib_path))
{
config.library_paths.push_back(lib_path.string());
}
}
// Auto-detect GPU compute capability using CUDA runtime
int device = 0;
if (cudaGetDevice(&device) == cudaSuccess)
{
cudaDeviceProp prop;
if (cudaGetDeviceProperties(&prop, device) == cudaSuccess)
{
int detected_sm = prop.major * 10 + prop.minor;
if (detected_sm >= 75)
{
config.sm_version = detected_sm;
}
}
}
if (config.sm_version == 0)
{
config.sm_version = 75;
}
config.optimization_level = 2;
config.debug = false;
config.verbose = false;
// Detect hostjit include path
if (const char* env = std::getenv("HOSTJIT_INCLUDE_PATH"))
{
config.hostjit_include_path = env;
}
#ifdef HOSTJIT_INCLUDE_DIR
else
{
config.hostjit_include_path = HOSTJIT_INCLUDE_DIR;
}
#endif
// Detect clang headers path. Build-time CLANG_HEADERS_DIR is the default;
// HOSTJIT_CLANG_PATH overrides it (e.g. for pip-installed wheels with a
// packaged copy of clang's CUDA headers).
if (const char* env = std::getenv("HOSTJIT_CLANG_PATH"))
{
config.clang_headers_path = env;
}
#ifdef CLANG_HEADERS_DIR
else
{
config.clang_headers_path = CLANG_HEADERS_DIR;
}
#endif
return config;
}
bool validateConfig(const CompilerConfig& config, std::string* error_message)
{
if (config.cuda_toolkit_path.empty())
{
if (error_message)
{
*error_message = "CUDA toolkit path not found. Please set CUDA_PATH or CUDA_HOME environment variable.";
}
return false;
}
if (!std::filesystem::exists(config.cuda_toolkit_path))
{
if (error_message)
{
*error_message = "CUDA toolkit path does not exist: " + config.cuda_toolkit_path;
}
return false;
}
std::filesystem::path cuda_h = std::filesystem::path(config.cuda_toolkit_path) / "include" / "cuda.h";
if (!std::filesystem::exists(cuda_h))
{
if (error_message)
{
*error_message = "CUDA headers not found at: " + cuda_h.string();
}
return false;
}
for (const auto& include_path : config.include_paths)
{
if (!std::filesystem::exists(include_path))
{
if (error_message)
{
*error_message = "Include path does not exist: " + include_path;
}
return false;
}
}
for (const auto& library_path : config.library_paths)
{
if (!std::filesystem::exists(library_path))
{
if (error_message)
{
*error_message = "Library path does not exist: " + library_path;
}
return false;
}
}
if (config.sm_version < 30 || config.sm_version > 150)
{
if (error_message)
{
*error_message = "Invalid SM version: " + std::to_string(config.sm_version) + " (must be between 30 and 150)";
}
return false;
}
if (config.optimization_level < 0 || config.optimization_level > 3)
{
if (error_message)
{
*error_message =
"Invalid optimization level: " + std::to_string(config.optimization_level) + " (must be between 0 and 3)";
}
return false;
}
return true;
}
} // namespace hostjit

56
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/codegen/bitcode.hpp Normal file
View File

@@ -0,0 +1,56 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
#include <cstdint>
#include <set>
#include <string>
#include <unordered_set>
#include <vector>
#include <cccl/c/types.h>
#include <hostjit/config.hpp>
namespace hostjit::codegen
{
// Manages bitcode files needed for linking. Collects LTOIR, LLVM IR,
// and C++ source (compiling the latter to bitcode on the fly).
// Tracks temp file paths for cleanup.
class BitcodeCollector
{
public:
explicit BitcodeCollector(CompilerConfig& config, uintptr_t unique_id);
// Add bitcode from an operator (handles LTOIR, LLVM_IR, CPP_SOURCE,
// and extra modules).
void add_op(cccl_op_t op, const std::string& label);
// Add bitcode from a custom iterator's advance/dereference ops.
void add_iterator(cccl_iterator_t it, const std::string& label_prefix);
// Returns true if the op has linked bitcode (LTOIR or LLVM_IR).
static bool is_bitcode_op(cccl_op_t op);
// Clean up all temporary files.
void cleanup();
private:
void add_raw_bitcode(const char* data, size_t size, const std::string& name);
bool compile_and_add(const char* source, size_t source_size, const std::string& name);
void add_op_code(cccl_op_t& op, const std::string& name);
CompilerConfig& config_;
uintptr_t unique_id_;
std::vector<std::string> temp_paths_;
std::set<std::string> added_symbols_; // dedup by op.name (when present)
std::unordered_set<std::size_t> added_content_hashes_; // dedup by content hash for unnamed extras
};
} // namespace hostjit::codegen

304
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/codegen/cub_call.hpp Normal file
View File

@@ -0,0 +1,304 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
#include <string>
#include <variant>
#include <vector>
#include <cccl/c/types.h>
#include <hostjit/config.hpp>
#include <hostjit/jit_compiler.hpp>
namespace hostjit::codegen
{
// Tags for non-cccl arguments (no runtime data, just control code generation)
struct temp_storage_t
{};
struct temp_bytes_t
{};
// num_items_t carries a name so the same tag type can express num_segments,
// num_needles, etc. — each becomes its own unsigned long long parameter.
struct num_items_t
{
const char* name = "num_items";
};
struct stream_t
{};
inline constexpr temp_storage_t temp_storage{};
inline constexpr temp_bytes_t temp_bytes{};
inline constexpr num_items_t num_items{};
inline constexpr num_items_t num_segments{"num_segments"};
inline constexpr num_items_t num_needles{"num_needles"};
inline constexpr num_items_t num_haystack{"num_haystack"};
inline constexpr stream_t stream{};
// Direction wrappers for iterators (cccl_iterator_t doesn't encode direction)
struct input_t
{
cccl_iterator_t it;
};
struct output_t
{
cccl_iterator_t it;
};
inline input_t in(cccl_iterator_t it)
{
return {it};
}
inline output_t out(cccl_iterator_t it)
{
return {it};
}
// cmp_t: wraps a cccl_op_t that should generate a comparison functor
// (bool operator()(const T&, const T&)) rather than the default binary reduce
// functor (T operator()(T, T)). Use cmp(op) where sort/search operators go.
struct cmp_t
{
cccl_op_t op;
};
inline cmp_t cmp(cccl_op_t op)
{
return {op};
}
// future_val_t: the init value lives on the device at runtime. Generates
// cub::FutureValue<accum_t>(static_cast<accum_t*>(param)) in the CUB call.
// Carries type info so find_accum_type can resolve accum_t correctly.
struct future_val_t
{
cccl_type_info type;
};
inline future_val_t future_val(cccl_type_info t)
{
return {t};
}
// unary_op_t: wraps a cccl_op_t used as a unary transform operator (T -> U).
// Carries the input/output type info so the functor can be typed correctly.
struct unary_op_t
{
cccl_op_t op;
cccl_type_info in_type;
cccl_type_info out_type;
};
inline unary_op_t unary_op(cccl_op_t op, cccl_type_info in_t, cccl_type_info out_t)
{
return {op, in_t, out_t};
}
// force_accum_type_t: overrides the accumulator type resolved by find_accum_type.
// Use when the natural accum type (first input) differs from the desired type.
// Generates no code — only influences type resolution.
struct force_accum_type_t
{
cccl_type_info type;
};
inline force_accum_type_t force_accum_type(cccl_type_info t)
{
return {t};
}
// pred(): shorthand for a unary bool predicate operator (e.g. for partition).
// Equivalent to unary_op with out_type = bool.
// Generates: bool operator()(const item_t& a) const { ... }
inline unary_op_t pred(cccl_op_t op, cccl_type_info item_t)
{
return {op, item_t, cccl_type_info{sizeof(bool), alignof(bool), CCCL_BOOLEAN}};
}
// typed_scalar_t: a by-value scalar of any cccl-known type, passed into the
// JIT wrapper as a host pointer and memcpy'd onto the stack before the CUB
// call. Use when the CUB API takes a small POD by value (e.g. radix_sort's
// `int begin_bit`, histogram's `int num_levels` / `level_t lower_level`).
// The caller supplies a void* host pointer to the value at the corresponding
// run-time arg position.
struct typed_scalar_t
{
cccl_type_info type;
const char* name;
};
inline typed_scalar_t typed_scalar(cccl_type_info t, const char* name)
{
return {t, name};
}
// env_stream_t: variant of stream_t that emits a cuda::std::execution::env
// wrapping a cuda::stream_ref instead of a bare cudaStream_t. Use with CUB
// algorithms that take an env (so CUB manages temp storage internally via
// the env's memory_resource — caller doesn't have to thread it through).
struct env_stream_t
{};
inline constexpr env_stream_t env_stream{};
// for_each_op_t: wraps a cccl_op_t with c.parallel's void op(T*) contract
// into the void op(T&) functor that cub::DeviceFor::ForEachN expects.
struct for_each_op_t
{
cccl_op_t op;
};
inline for_each_op_t for_each_op(cccl_op_t op)
{
return {op};
}
// double_buffer_t: constructs a cub::DoubleBuffer<elem_t> from two host
// pointers (one "in" buffer, one "out" buffer) and passes it to the CUB call.
// Used by the DoubleBuffer overloads of DeviceRadixSort / DeviceSegmentedSort
// where the caller is willing to let CUB swap buffers and report the final
// location via the buffer's `selector` member. var_name controls the C++ name
// of the generated local — pair a `selector_out_t` with the same name to read
// `<var_name>.selector` after the call.
struct double_buffer_t
{
cccl_iterator_t in_it;
cccl_iterator_t out_it;
const char* var_name;
};
inline double_buffer_t double_buffer(cccl_iterator_t in_it, cccl_iterator_t out_it, const char* var_name = "d_buffer")
{
return {in_it, out_it, var_name};
}
// selector_out_t: emits a `void* selector_out` parameter and, after the CUB
// call, writes `<buffer_var_name>.selector` to it. Must be paired with a
// double_buffer_t whose var_name matches.
struct selector_out_t
{
const char* buffer_var_name;
};
inline selector_out_t selector_out(const char* buffer_var_name = "d_buffer")
{
return {buffer_var_name};
}
// Argument variant: everything that can appear in .with()
using Arg = std::variant<
temp_storage_t,
temp_bytes_t,
num_items_t,
stream_t,
env_stream_t,
input_t,
output_t,
cccl_op_t,
cmp_t,
unary_op_t,
for_each_op_t,
double_buffer_t,
selector_out_t,
future_val_t,
cccl_value_t,
force_accum_type_t,
typed_scalar_t>;
// Result of a successful single-function compilation.
struct CubCallResult
{
JITCompiler* compiler; // caller takes ownership
void* fn_ptr; // the exported function
std::vector<char> cubin; // for SASS inspection
};
// Result of a successful multi-function compilation (one TU, N functions).
struct MultiCubCallResult
{
JITCompiler* compiler; // caller takes ownership; one compiler for the whole TU
std::vector<char> cubin; // single cubin for the whole TU
std::vector<void*> fn_ptrs; // exported functions in the same order as the input CubCalls
};
class CubCall
{
public:
// Start building: specify the CUB header to include.
static CubCall from(const char* include_header);
// Specify the CUB function to call (e.g., "cub::DeviceReduce::Reduce").
CubCall& run(const char* cub_function);
// Optionally override the exported function name (default: "cccl_jit_fn").
CubCall& name(const char* export_name);
// Add arguments in CUB call order. Each argument is dispatched by type.
template <typename... Args>
CubCall& with(Args&&... args)
{
(args_.emplace_back(Arg{std::forward<Args>(args)}), ...);
return *this;
}
// Wrap all input iterators in cuda::std::make_tuple() in the generated CUB call.
// Required for cub::DeviceTransform::Transform with multiple inputs.
CubCall& use_tuple_inputs()
{
tuple_inputs_ = true;
return *this;
}
// Generate the complete CUDA source string (useful for debugging).
std::string source() const;
// Compile the generated source and return the function pointer.
CubCallResult compile(
int cc_major,
int cc_minor,
cccl_build_config* config = nullptr,
const char* ctk_path = nullptr,
const char* cccl_include_path = nullptr) const;
// Compile multiple CubCalls into a single translation unit. One Clang
// invocation, one cubin, one JITCompiler; each function is dlsym'd by its
// .name(...) and returned in the input order. All CubCalls must share the
// same CUB include header (.from(...)). Per-function preambles are isolated
// inside `namespace fn_<i> { ... }` blocks; extern "C" symbols escape the
// namespace and stay globally dlsym-able.
static MultiCubCallResult compile(
std::initializer_list<CubCall> calls,
int cc_major,
int cc_minor,
cccl_build_config* config = nullptr,
const char* ctk_path = nullptr,
const char* cccl_include_path = nullptr);
private:
std::string include_;
std::string cub_function_;
std::string fn_name_ = "cccl_jit_fn";
std::vector<Arg> args_;
bool tuple_inputs_ = false;
// Internal: just the per-function body (preamble + function defn), no
// shared #includes. Used by the multi-compile path to wrap N bodies in
// N namespaces under a single shared include block.
std::string body() const;
// Internal: walk args_ and register any user-op / iterator bitcode with
// the given collector. Factored out so the multi-compile path can share
// one collector across several CubCalls.
void collect_bitcode(class BitcodeCollector& bitcode, int& op_idx, int& in_idx, int& out_idx) const;
// Internal: builds a hostjit::CompilerConfig from the standard cc + paths +
// build_config inputs every compile entry point takes. Shared between the
// single-fn and multi-fn compile overloads so flag/path handling lives in
// one place.
static hostjit::CompilerConfig make_jit_config(
int cc_major,
int cc_minor,
cccl_build_config* config,
const char* ctk_path,
const char* cccl_include_path,
const std::string& entry_point_name);
};
} // namespace hostjit::codegen

50
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/codegen/iterators.hpp Normal file
View File

@@ -0,0 +1,50 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
#include <string>
#include <cccl/c/types.h>
namespace hostjit::codegen
{
// Result of generating iterator code.
struct IteratorCode
{
std::string preamble; // type alias or struct definition (goes at file scope)
std::string setup_code; // initialization inside function body
std::string local_var; // e.g., "in_0"
std::string type_name; // e.g., "in_0_it_t" or "accum_t*"
};
// Generate code for an input iterator.
// For CCCL_POINTER: emits a type alias and pointer cast.
// For CCCL_ITERATOR: emits a full iterator struct with advance/dereference.
IteratorCode make_input_iterator(
cccl_iterator_t it,
const std::string& value_type_name, // resolved C++ type of iterator's value
const std::string& accum_type_name, // accumulator type alias (for pointer fallback)
const std::string& struct_name, // e.g., "in_0_it_t"
const std::string& var_name, // e.g., "in_0"
const std::string& state_param); // e.g., "d_in_0" (void* param name)
// Generate code for an output iterator.
// value_type_name: if non-empty, overrides accum_type_name as the element type
// for the pointer/proxy. Use this when the output element type differs from the
// accumulator (e.g. item values in a key-value sort).
IteratorCode make_output_iterator(
cccl_iterator_t it,
const std::string& accum_type_name,
const std::string& struct_name,
const std::string& var_name,
const std::string& state_param,
const std::string& value_type_name = "");
} // namespace hostjit::codegen

71
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/codegen/operators.hpp Normal file
View File

@@ -0,0 +1,71 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
#include <string>
#include <cccl/c/types.h>
namespace hostjit::codegen
{
// Result of generating operator code.
struct OperatorCode
{
std::string preamble; // extern decl + functor struct (goes at file scope)
std::string setup_code; // initialization inside function body
std::string local_var; // e.g., "op_0"
};
// Generate code for a binary operator (reduce, scan).
// Produces an extern "C" device function declaration (or inline for well-known ops)
// and a functor struct that wraps it.
OperatorCode make_binary_op(
cccl_op_t op,
const std::string& accum_type, // C++ type name for operands
const std::string& functor_name, // e.g., "ReduceOp"
const std::string& var_name, // e.g., "op_0"
const std::string& state_param, // e.g., "op_0_state" (void* param name)
bool has_bitcode);
// Generate code for a unary operator (transform).
// Produces a functor with operator()(const in_type& a) const -> out_type.
OperatorCode make_unary_op(
cccl_op_t op,
const std::string& in_type, // C++ type name for input operand
const std::string& out_type, // C++ type name for result
const std::string& functor_name, // e.g., "UnaryOp"
const std::string& var_name, // e.g., "op_0"
const std::string& state_param, // e.g., "op_0_state" (void* param name)
bool has_bitcode);
// Generate code for a comparison operator (sort).
// Same as binary op but the functor returns bool.
OperatorCode make_comparison_op(
cccl_op_t op,
const std::string& key_type, // C++ type name for keys
const std::string& functor_name, // e.g., "CompareOp"
const std::string& var_name, // e.g., "cmp_0"
const std::string& state_param, // e.g., "cmp_0_state"
bool has_bitcode);
// Generate code for a for_each operator. Adapts c.parallel's user-op contract
// (`void op(T*)`) to the contract that cub::DeviceFor::ForEachN expects
// (`void op(T&)`). Functor is stateless for non-stateful ops; for stateful
// ops it embeds the state bytes inline so they ride along into device
// constant memory via the kernel-arg copy.
OperatorCode make_for_each_op(
cccl_op_t op,
const std::string& elem_type, // C++ type name for the iterator's element
const std::string& functor_name, // e.g., "ForEachOp"
const std::string& var_name, // e.g., "op_0"
const std::string& state_param, // e.g., "op_0_state"
bool has_bitcode);
} // namespace hostjit::codegen

32
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/codegen/types.hpp Normal file
View File

@@ -0,0 +1,32 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
#include <string>
#include <cccl/c/types.h>
namespace hostjit::codegen
{
// Maps cccl_type_enum to plain C/C++ type names (e.g., "int", "float").
// Returns "" for CCCL_STORAGE (caller must handle custom types).
std::string get_type_name(cccl_type_enum type);
// Generates an aligned storage struct definition.
// Example: "struct __align__(8) my_storage_t {\n char data[16];\n};\n"
std::string make_storage_type(const char* name, size_t size, size_t alignment);
// Returns the C++ type name for a cccl_type_info.
// For known types, returns the type name directly.
// For CCCL_STORAGE, emits a storage struct definition into `out_preamble`
// and returns `fallback_alias`.
std::string resolve_type(cccl_type_info info, const char* fallback_alias, std::string& out_preamble);
} // namespace hostjit::codegen

54
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/compiler.hpp Normal file
View File

@@ -0,0 +1,54 @@
#pragma once
#include <string>
#include <vector>
namespace hostjit
{
struct CompilationResult
{
bool success;
std::string object_file_path; // Path to generated .o file
std::string diagnostics; // Compiler messages
std::vector<char> cubin; // Device cubin extracted during compilation
};
struct BitcodeResult
{
bool success;
std::string bitcode; // LLVM bitcode bytes
std::string diagnostics;
};
struct LinkResult
{
bool success;
std::string library_path; // Path to .so file
std::string diagnostics;
};
// Forward declaration to avoid including heavy Clang headers
struct CompilerConfig;
class CUDACompiler
{
public:
CUDACompiler();
~CUDACompiler();
// Compile CUDA device source to LLVM bitcode
BitcodeResult compileToDeviceBitcode(const std::string& source_code, const CompilerConfig& config);
// Compile CUDA source code to object file
CompilationResult
compileToObject(const std::string& source_code, const std::string& output_path, const CompilerConfig& config);
// Link object files to shared library
LinkResult linkToSharedLibrary(
const std::vector<std::string>& object_files, const std::string& output_path, const CompilerConfig& config);
private:
class Impl;
Impl* impl_;
};
} // namespace hostjit

36
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/config.hpp Normal file
View File

@@ -0,0 +1,36 @@
#pragma once
#include <string>
#include <unordered_map>
#include <vector>
namespace hostjit
{
struct CompilerConfig
{
std::string cuda_toolkit_path;
std::string hostjit_include_path; // Path to hostjit include directory (for minimal CUDA runtime)
std::string clang_headers_path; // Path to Clang's built-in CUDA headers (overrides CLANG_HEADERS_DIR)
std::string cccl_include_path; // Path to CCCL headers (overrides CCCL_SOURCE_DIR); contains cub/, thrust/, cuda/
std::vector<std::string> include_paths;
std::vector<std::string> library_paths;
std::vector<std::string> device_bitcode_files; // Raw LLVM bitcode (magic "BC") linked via LLVM's Linker
std::vector<std::string> device_ltoir_files; // NVRTC LTOIR; linked at the nvJitLink stage with -lto
std::unordered_map<std::string, std::string> macro_definitions; // key=macro name, value=macro value (empty for flag
// macros)
int sm_version = 70;
int optimization_level = 2;
bool debug = false;
bool verbose = false;
bool trace_includes = false; // Show all included headers during compilation (for debugging header search)
bool keep_artifacts = false; // Keep compiled artifacts for inspection (PTX, object files, etc.)
std::string entry_point_name; // Name of the exported entry point function (used for post-link optimization)
bool enable_pch = false; // Cache precompiled headers on disk to speed up repeated builds
};
// Auto-detect CUDA toolkit and create default configuration
CompilerConfig detectDefaultConfig();
// Validate that the configuration is usable
bool validateConfig(const CompilerConfig& config, std::string* error_message = nullptr);
} // namespace hostjit

1716
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/__clang_cuda_device_functions.h Normal file
View File

File diff suppressed because it is too large Load Diff

371
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/__clang_cuda_libdevice_declares.h Normal file
View File

@@ -0,0 +1,371 @@
/*===-- __clang_cuda_libdevice_declares.h - decls for libdevice functions --===
*
* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
* See https://llvm.org/LICENSE.txt for license information.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*
*===-----------------------------------------------------------------------===
*/
#ifndef __CLANG_CUDA_LIBDEVICE_DECLARES_H__
#define __CLANG_CUDA_LIBDEVICE_DECLARES_H__
#if defined(__cplusplus)
extern "C" {
#endif
#define __DEVICE__ __device__
__DEVICE__ int __nv_abs(int __a);
__DEVICE__ double __nv_acos(double __a);
__DEVICE__ float __nv_acosf(float __a);
__DEVICE__ double __nv_acosh(double __a);
__DEVICE__ float __nv_acoshf(float __a);
__DEVICE__ double __nv_asin(double __a);
__DEVICE__ float __nv_asinf(float __a);
__DEVICE__ double __nv_asinh(double __a);
__DEVICE__ float __nv_asinhf(float __a);
__DEVICE__ double __nv_atan2(double __a, double __b);
__DEVICE__ float __nv_atan2f(float __a, float __b);
__DEVICE__ double __nv_atan(double __a);
__DEVICE__ float __nv_atanf(float __a);
__DEVICE__ double __nv_atanh(double __a);
__DEVICE__ float __nv_atanhf(float __a);
__DEVICE__ int __nv_brev(int __a);
__DEVICE__ long long __nv_brevll(long long __a);
__DEVICE__ int __nv_byte_perm(int __a, int __b, int __c);
__DEVICE__ double __nv_cbrt(double __a);
__DEVICE__ float __nv_cbrtf(float __a);
__DEVICE__ double __nv_ceil(double __a);
__DEVICE__ float __nv_ceilf(float __a);
__DEVICE__ int __nv_clz(int __a);
__DEVICE__ int __nv_clzll(long long __a);
__DEVICE__ double __nv_copysign(double __a, double __b);
__DEVICE__ float __nv_copysignf(float __a, float __b);
__DEVICE__ double __nv_cos(double __a);
__DEVICE__ float __nv_cosf(float __a);
__DEVICE__ double __nv_cosh(double __a);
__DEVICE__ float __nv_coshf(float __a);
__DEVICE__ double __nv_cospi(double __a);
__DEVICE__ float __nv_cospif(float __a);
__DEVICE__ double __nv_cyl_bessel_i0(double __a);
__DEVICE__ float __nv_cyl_bessel_i0f(float __a);
__DEVICE__ double __nv_cyl_bessel_i1(double __a);
__DEVICE__ float __nv_cyl_bessel_i1f(float __a);
__DEVICE__ double __nv_dadd_rd(double __a, double __b);
__DEVICE__ double __nv_dadd_rn(double __a, double __b);
__DEVICE__ double __nv_dadd_ru(double __a, double __b);
__DEVICE__ double __nv_dadd_rz(double __a, double __b);
__DEVICE__ double __nv_ddiv_rd(double __a, double __b);
__DEVICE__ double __nv_ddiv_rn(double __a, double __b);
__DEVICE__ double __nv_ddiv_ru(double __a, double __b);
__DEVICE__ double __nv_ddiv_rz(double __a, double __b);
__DEVICE__ double __nv_dmul_rd(double __a, double __b);
__DEVICE__ double __nv_dmul_rn(double __a, double __b);
__DEVICE__ double __nv_dmul_ru(double __a, double __b);
__DEVICE__ double __nv_dmul_rz(double __a, double __b);
__DEVICE__ float __nv_double2float_rd(double __a);
__DEVICE__ float __nv_double2float_rn(double __a);
__DEVICE__ float __nv_double2float_ru(double __a);
__DEVICE__ float __nv_double2float_rz(double __a);
__DEVICE__ int __nv_double2hiint(double __a);
__DEVICE__ int __nv_double2int_rd(double __a);
__DEVICE__ int __nv_double2int_rn(double __a);
__DEVICE__ int __nv_double2int_ru(double __a);
__DEVICE__ int __nv_double2int_rz(double __a);
__DEVICE__ long long __nv_double2ll_rd(double __a);
__DEVICE__ long long __nv_double2ll_rn(double __a);
__DEVICE__ long long __nv_double2ll_ru(double __a);
__DEVICE__ long long __nv_double2ll_rz(double __a);
__DEVICE__ int __nv_double2loint(double __a);
__DEVICE__ unsigned int __nv_double2uint_rd(double __a);
__DEVICE__ unsigned int __nv_double2uint_rn(double __a);
__DEVICE__ unsigned int __nv_double2uint_ru(double __a);
__DEVICE__ unsigned int __nv_double2uint_rz(double __a);
__DEVICE__ unsigned long long __nv_double2ull_rd(double __a);
__DEVICE__ unsigned long long __nv_double2ull_rn(double __a);
__DEVICE__ unsigned long long __nv_double2ull_ru(double __a);
__DEVICE__ unsigned long long __nv_double2ull_rz(double __a);
__DEVICE__ unsigned long long __nv_double_as_longlong(double __a);
__DEVICE__ double __nv_drcp_rd(double __a);
__DEVICE__ double __nv_drcp_rn(double __a);
__DEVICE__ double __nv_drcp_ru(double __a);
__DEVICE__ double __nv_drcp_rz(double __a);
__DEVICE__ double __nv_dsqrt_rd(double __a);
__DEVICE__ double __nv_dsqrt_rn(double __a);
__DEVICE__ double __nv_dsqrt_ru(double __a);
__DEVICE__ double __nv_dsqrt_rz(double __a);
__DEVICE__ double __nv_dsub_rd(double __a, double __b);
__DEVICE__ double __nv_dsub_rn(double __a, double __b);
__DEVICE__ double __nv_dsub_ru(double __a, double __b);
__DEVICE__ double __nv_dsub_rz(double __a, double __b);
__DEVICE__ double __nv_erfc(double __a);
__DEVICE__ float __nv_erfcf(float __a);
__DEVICE__ double __nv_erfcinv(double __a);
__DEVICE__ float __nv_erfcinvf(float __a);
__DEVICE__ double __nv_erfcx(double __a);
__DEVICE__ float __nv_erfcxf(float __a);
__DEVICE__ double __nv_erf(double __a);
__DEVICE__ float __nv_erff(float __a);
__DEVICE__ double __nv_erfinv(double __a);
__DEVICE__ float __nv_erfinvf(float __a);
__DEVICE__ double __nv_exp10(double __a);
__DEVICE__ float __nv_exp10f(float __a);
__DEVICE__ double __nv_exp2(double __a);
__DEVICE__ float __nv_exp2f(float __a);
__DEVICE__ double __nv_exp(double __a);
__DEVICE__ float __nv_expf(float __a);
__DEVICE__ double __nv_expm1(double __a);
__DEVICE__ float __nv_expm1f(float __a);
__DEVICE__ double __nv_fabs(double __a);
__DEVICE__ float __nv_fabsf(float __a);
__DEVICE__ float __nv_fadd_rd(float __a, float __b);
__DEVICE__ float __nv_fadd_rn(float __a, float __b);
__DEVICE__ float __nv_fadd_ru(float __a, float __b);
__DEVICE__ float __nv_fadd_rz(float __a, float __b);
__DEVICE__ float __nv_fast_cosf(float __a);
__DEVICE__ float __nv_fast_exp10f(float __a);
__DEVICE__ float __nv_fast_expf(float __a);
__DEVICE__ float __nv_fast_fdividef(float __a, float __b);
__DEVICE__ float __nv_fast_log10f(float __a);
__DEVICE__ float __nv_fast_log2f(float __a);
__DEVICE__ float __nv_fast_logf(float __a);
__DEVICE__ float __nv_fast_powf(float __a, float __b);
__DEVICE__ void __nv_fast_sincosf(float __a, float* __s, float* __c);
__DEVICE__ float __nv_fast_sinf(float __a);
__DEVICE__ float __nv_fast_tanf(float __a);
__DEVICE__ double __nv_fdim(double __a, double __b);
__DEVICE__ float __nv_fdimf(float __a, float __b);
__DEVICE__ float __nv_fdiv_rd(float __a, float __b);
__DEVICE__ float __nv_fdiv_rn(float __a, float __b);
__DEVICE__ float __nv_fdiv_ru(float __a, float __b);
__DEVICE__ float __nv_fdiv_rz(float __a, float __b);
__DEVICE__ int __nv_ffs(int __a);
__DEVICE__ int __nv_ffsll(long long __a);
__DEVICE__ int __nv_finitef(float __a);
__DEVICE__ unsigned short __nv_float2half_rn(float __a);
__DEVICE__ int __nv_float2int_rd(float __a);
__DEVICE__ int __nv_float2int_rn(float __a);
__DEVICE__ int __nv_float2int_ru(float __a);
__DEVICE__ int __nv_float2int_rz(float __a);
__DEVICE__ long long __nv_float2ll_rd(float __a);
__DEVICE__ long long __nv_float2ll_rn(float __a);
__DEVICE__ long long __nv_float2ll_ru(float __a);
__DEVICE__ long long __nv_float2ll_rz(float __a);
__DEVICE__ unsigned int __nv_float2uint_rd(float __a);
__DEVICE__ unsigned int __nv_float2uint_rn(float __a);
__DEVICE__ unsigned int __nv_float2uint_ru(float __a);
__DEVICE__ unsigned int __nv_float2uint_rz(float __a);
__DEVICE__ unsigned long long __nv_float2ull_rd(float __a);
__DEVICE__ unsigned long long __nv_float2ull_rn(float __a);
__DEVICE__ unsigned long long __nv_float2ull_ru(float __a);
__DEVICE__ unsigned long long __nv_float2ull_rz(float __a);
__DEVICE__ int __nv_float_as_int(float __a);
__DEVICE__ unsigned int __nv_float_as_uint(float __a);
__DEVICE__ double __nv_floor(double __a);
__DEVICE__ float __nv_floorf(float __a);
__DEVICE__ double __nv_fma(double __a, double __b, double __c);
__DEVICE__ float __nv_fmaf(float __a, float __b, float __c);
__DEVICE__ float __nv_fmaf_ieee_rd(float __a, float __b, float __c);
__DEVICE__ float __nv_fmaf_ieee_rn(float __a, float __b, float __c);
__DEVICE__ float __nv_fmaf_ieee_ru(float __a, float __b, float __c);
__DEVICE__ float __nv_fmaf_ieee_rz(float __a, float __b, float __c);
__DEVICE__ float __nv_fmaf_rd(float __a, float __b, float __c);
__DEVICE__ float __nv_fmaf_rn(float __a, float __b, float __c);
__DEVICE__ float __nv_fmaf_ru(float __a, float __b, float __c);
__DEVICE__ float __nv_fmaf_rz(float __a, float __b, float __c);
__DEVICE__ double __nv_fma_rd(double __a, double __b, double __c);
__DEVICE__ double __nv_fma_rn(double __a, double __b, double __c);
__DEVICE__ double __nv_fma_ru(double __a, double __b, double __c);
__DEVICE__ double __nv_fma_rz(double __a, double __b, double __c);
__DEVICE__ double __nv_fmax(double __a, double __b);
__DEVICE__ float __nv_fmaxf(float __a, float __b);
__DEVICE__ double __nv_fmin(double __a, double __b);
__DEVICE__ float __nv_fminf(float __a, float __b);
__DEVICE__ double __nv_fmod(double __a, double __b);
__DEVICE__ float __nv_fmodf(float __a, float __b);
__DEVICE__ float __nv_fmul_rd(float __a, float __b);
__DEVICE__ float __nv_fmul_rn(float __a, float __b);
__DEVICE__ float __nv_fmul_ru(float __a, float __b);
__DEVICE__ float __nv_fmul_rz(float __a, float __b);
__DEVICE__ float __nv_frcp_rd(float __a);
__DEVICE__ float __nv_frcp_rn(float __a);
__DEVICE__ float __nv_frcp_ru(float __a);
__DEVICE__ float __nv_frcp_rz(float __a);
__DEVICE__ double __nv_frexp(double __a, int* __b);
__DEVICE__ float __nv_frexpf(float __a, int* __b);
__DEVICE__ float __nv_frsqrt_rn(float __a);
__DEVICE__ float __nv_fsqrt_rd(float __a);
__DEVICE__ float __nv_fsqrt_rn(float __a);
__DEVICE__ float __nv_fsqrt_ru(float __a);
__DEVICE__ float __nv_fsqrt_rz(float __a);
__DEVICE__ float __nv_fsub_rd(float __a, float __b);
__DEVICE__ float __nv_fsub_rn(float __a, float __b);
__DEVICE__ float __nv_fsub_ru(float __a, float __b);
__DEVICE__ float __nv_fsub_rz(float __a, float __b);
__DEVICE__ int __nv_hadd(int __a, int __b);
__DEVICE__ float __nv_half2float(unsigned short __h);
__DEVICE__ double __nv_hiloint2double(int __a, int __b);
__DEVICE__ double __nv_hypot(double __a, double __b);
__DEVICE__ float __nv_hypotf(float __a, float __b);
__DEVICE__ int __nv_ilogb(double __a);
__DEVICE__ int __nv_ilogbf(float __a);
__DEVICE__ double __nv_int2double_rn(int __a);
__DEVICE__ float __nv_int2float_rd(int __a);
__DEVICE__ float __nv_int2float_rn(int __a);
__DEVICE__ float __nv_int2float_ru(int __a);
__DEVICE__ float __nv_int2float_rz(int __a);
__DEVICE__ float __nv_int_as_float(int __a);
__DEVICE__ int __nv_isfinited(double __a);
__DEVICE__ int __nv_isinfd(double __a);
__DEVICE__ int __nv_isinff(float __a);
__DEVICE__ int __nv_isnand(double __a);
__DEVICE__ int __nv_isnanf(float __a);
__DEVICE__ double __nv_j0(double __a);
__DEVICE__ float __nv_j0f(float __a);
__DEVICE__ double __nv_j1(double __a);
__DEVICE__ float __nv_j1f(float __a);
__DEVICE__ float __nv_jnf(int __a, float __b);
__DEVICE__ double __nv_jn(int __a, double __b);
__DEVICE__ double __nv_ldexp(double __a, int __b);
__DEVICE__ float __nv_ldexpf(float __a, int __b);
__DEVICE__ double __nv_lgamma(double __a);
__DEVICE__ float __nv_lgammaf(float __a);
__DEVICE__ double __nv_ll2double_rd(long long __a);
__DEVICE__ double __nv_ll2double_rn(long long __a);
__DEVICE__ double __nv_ll2double_ru(long long __a);
__DEVICE__ double __nv_ll2double_rz(long long __a);
__DEVICE__ float __nv_ll2float_rd(long long __a);
__DEVICE__ float __nv_ll2float_rn(long long __a);
__DEVICE__ float __nv_ll2float_ru(long long __a);
__DEVICE__ float __nv_ll2float_rz(long long __a);
__DEVICE__ long long __nv_llabs(long long __a);
__DEVICE__ long long __nv_llmax(long long __a, long long __b);
__DEVICE__ long long __nv_llmin(long long __a, long long __b);
__DEVICE__ long long __nv_llrint(double __a);
__DEVICE__ long long __nv_llrintf(float __a);
__DEVICE__ long long __nv_llround(double __a);
__DEVICE__ long long __nv_llroundf(float __a);
__DEVICE__ double __nv_log10(double __a);
__DEVICE__ float __nv_log10f(float __a);
__DEVICE__ double __nv_log1p(double __a);
__DEVICE__ float __nv_log1pf(float __a);
__DEVICE__ double __nv_log2(double __a);
__DEVICE__ float __nv_log2f(float __a);
__DEVICE__ double __nv_logb(double __a);
__DEVICE__ float __nv_logbf(float __a);
__DEVICE__ double __nv_log(double __a);
__DEVICE__ float __nv_logf(float __a);
__DEVICE__ double __nv_longlong_as_double(long long __a);
__DEVICE__ int __nv_max(int __a, int __b);
__DEVICE__ int __nv_min(int __a, int __b);
__DEVICE__ double __nv_modf(double __a, double* __b);
__DEVICE__ float __nv_modff(float __a, float* __b);
__DEVICE__ int __nv_mul24(int __a, int __b);
__DEVICE__ long long __nv_mul64hi(long long __a, long long __b);
__DEVICE__ int __nv_mulhi(int __a, int __b);
__DEVICE__ double __nv_nan(const signed char* __a);
__DEVICE__ float __nv_nanf(const signed char* __a);
__DEVICE__ double __nv_nearbyint(double __a);
__DEVICE__ float __nv_nearbyintf(float __a);
__DEVICE__ double __nv_nextafter(double __a, double __b);
__DEVICE__ float __nv_nextafterf(float __a, float __b);
__DEVICE__ double __nv_norm3d(double __a, double __b, double __c);
__DEVICE__ float __nv_norm3df(float __a, float __b, float __c);
__DEVICE__ double __nv_norm4d(double __a, double __b, double __c, double __d);
__DEVICE__ float __nv_norm4df(float __a, float __b, float __c, float __d);
__DEVICE__ double __nv_normcdf(double __a);
__DEVICE__ float __nv_normcdff(float __a);
__DEVICE__ double __nv_normcdfinv(double __a);
__DEVICE__ float __nv_normcdfinvf(float __a);
__DEVICE__ float __nv_normf(int __a, const float* __b);
__DEVICE__ double __nv_norm(int __a, const double* __b);
__DEVICE__ int __nv_popc(unsigned int __a);
__DEVICE__ int __nv_popcll(unsigned long long __a);
__DEVICE__ double __nv_pow(double __a, double __b);
__DEVICE__ float __nv_powf(float __a, float __b);
__DEVICE__ double __nv_powi(double __a, int __b);
__DEVICE__ float __nv_powif(float __a, int __b);
__DEVICE__ double __nv_rcbrt(double __a);
__DEVICE__ float __nv_rcbrtf(float __a);
__DEVICE__ double __nv_rcp64h(double __a);
__DEVICE__ double __nv_remainder(double __a, double __b);
__DEVICE__ float __nv_remainderf(float __a, float __b);
__DEVICE__ double __nv_remquo(double __a, double __b, int* __c);
__DEVICE__ float __nv_remquof(float __a, float __b, int* __c);
__DEVICE__ int __nv_rhadd(int __a, int __b);
__DEVICE__ double __nv_rhypot(double __a, double __b);
__DEVICE__ float __nv_rhypotf(float __a, float __b);
__DEVICE__ double __nv_rint(double __a);
__DEVICE__ float __nv_rintf(float __a);
__DEVICE__ double __nv_rnorm3d(double __a, double __b, double __c);
__DEVICE__ float __nv_rnorm3df(float __a, float __b, float __c);
__DEVICE__ double __nv_rnorm4d(double __a, double __b, double __c, double __d);
__DEVICE__ float __nv_rnorm4df(float __a, float __b, float __c, float __d);
__DEVICE__ float __nv_rnormf(int __a, const float* __b);
__DEVICE__ double __nv_rnorm(int __a, const double* __b);
__DEVICE__ double __nv_round(double __a);
__DEVICE__ float __nv_roundf(float __a);
__DEVICE__ double __nv_rsqrt(double __a);
__DEVICE__ float __nv_rsqrtf(float __a);
__DEVICE__ int __nv_sad(int __a, int __b, int __c);
__DEVICE__ float __nv_saturatef(float __a);
__DEVICE__ double __nv_scalbn(double __a, int __b);
__DEVICE__ float __nv_scalbnf(float __a, int __b);
__DEVICE__ int __nv_signbitd(double __a);
__DEVICE__ int __nv_signbitf(float __a);
__DEVICE__ void __nv_sincos(double __a, double* __b, double* __c);
__DEVICE__ void __nv_sincosf(float __a, float* __b, float* __c);
__DEVICE__ void __nv_sincospi(double __a, double* __b, double* __c);
__DEVICE__ void __nv_sincospif(float __a, float* __b, float* __c);
__DEVICE__ double __nv_sin(double __a);
__DEVICE__ float __nv_sinf(float __a);
__DEVICE__ double __nv_sinh(double __a);
__DEVICE__ float __nv_sinhf(float __a);
__DEVICE__ double __nv_sinpi(double __a);
__DEVICE__ float __nv_sinpif(float __a);
__DEVICE__ double __nv_sqrt(double __a);
__DEVICE__ float __nv_sqrtf(float __a);
__DEVICE__ double __nv_tan(double __a);
__DEVICE__ float __nv_tanf(float __a);
__DEVICE__ double __nv_tanh(double __a);
__DEVICE__ float __nv_tanhf(float __a);
__DEVICE__ double __nv_tgamma(double __a);
__DEVICE__ float __nv_tgammaf(float __a);
__DEVICE__ double __nv_trunc(double __a);
__DEVICE__ float __nv_truncf(float __a);
__DEVICE__ int __nv_uhadd(unsigned int __a, unsigned int __b);
__DEVICE__ double __nv_uint2double_rn(unsigned int __i);
__DEVICE__ float __nv_uint2float_rd(unsigned int __a);
__DEVICE__ float __nv_uint2float_rn(unsigned int __a);
__DEVICE__ float __nv_uint2float_ru(unsigned int __a);
__DEVICE__ float __nv_uint2float_rz(unsigned int __a);
__DEVICE__ float __nv_uint_as_float(unsigned int __a);
__DEVICE__ double __nv_ull2double_rd(unsigned long long __a);
__DEVICE__ double __nv_ull2double_rn(unsigned long long __a);
__DEVICE__ double __nv_ull2double_ru(unsigned long long __a);
__DEVICE__ double __nv_ull2double_rz(unsigned long long __a);
__DEVICE__ float __nv_ull2float_rd(unsigned long long __a);
__DEVICE__ float __nv_ull2float_rn(unsigned long long __a);
__DEVICE__ float __nv_ull2float_ru(unsigned long long __a);
__DEVICE__ float __nv_ull2float_rz(unsigned long long __a);
__DEVICE__ unsigned long long __nv_ullmax(unsigned long long __a, unsigned long long __b);
__DEVICE__ unsigned long long __nv_ullmin(unsigned long long __a, unsigned long long __b);
__DEVICE__ unsigned int __nv_umax(unsigned int __a, unsigned int __b);
__DEVICE__ unsigned int __nv_umin(unsigned int __a, unsigned int __b);
__DEVICE__ unsigned int __nv_umul24(unsigned int __a, unsigned int __b);
__DEVICE__ unsigned long long __nv_umul64hi(unsigned long long __a, unsigned long long __b);
__DEVICE__ unsigned int __nv_umulhi(unsigned int __a, unsigned int __b);
__DEVICE__ unsigned int __nv_urhadd(unsigned int __a, unsigned int __b);
__DEVICE__ unsigned int __nv_usad(unsigned int __a, unsigned int __b, unsigned int __c);
__DEVICE__ double __nv_y0(double __a);
__DEVICE__ float __nv_y0f(float __a);
__DEVICE__ double __nv_y1(double __a);
__DEVICE__ float __nv_y1f(float __a);
__DEVICE__ float __nv_ynf(int __a, float __b);
__DEVICE__ double __nv_yn(int __a, double __b);
#if defined(__cplusplus)
} // extern "C"
#endif
#endif // __CLANG_CUDA_LIBDEVICE_DECLARES_H__

809
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/__clang_cuda_math.h Normal file
View File

@@ -0,0 +1,809 @@
/*===---- __clang_cuda_math.h - Device-side CUDA math support --------------===
*
* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
* See https://llvm.org/LICENSE.txt for license information.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*
*===-----------------------------------------------------------------------===
*/
#ifndef __CLANG_CUDA_MATH_H__
#define __CLANG_CUDA_MATH_H__
#ifndef __CUDA__
# error "This file is for CUDA compilation only."
#endif
// The __CLANG_GPU_DISABLE_MATH_WRAPPERS macro provides a way to let standard
// libcalls reach the link step instead of being eagerly replaced.
#ifndef __CLANG_GPU_DISABLE_MATH_WRAPPERS
// __DEVICE__ is a helper macro with common set of attributes for the wrappers
// we implement in this file. We need static in order to avoid emitting unused
// functions and __forceinline__ helps inlining these wrappers at -O1.
# pragma push_macro("__DEVICE__")
# define __DEVICE__ static __device__ __forceinline__
// Specialized version of __DEVICE__ for functions with void return type.
# pragma push_macro("__DEVICE_VOID__")
# define __DEVICE_VOID__ __DEVICE__
// libdevice provides fast low precision and slow full-recision implementations
// for some functions. Which one gets selected depends on
// __CLANG_CUDA_APPROX_TRANSCENDENTALS__ which gets defined by clang if
// -ffast-math or -fgpu-approx-transcendentals are in effect.
# pragma push_macro("__FAST_OR_SLOW")
# if defined(__CLANG_GPU_APPROX_TRANSCENDENTALS__)
# define __FAST_OR_SLOW(fast, slow) fast
# else
# define __FAST_OR_SLOW(fast, slow) slow
# endif
__DEVICE__ int abs(int __a)
{
return __nv_abs(__a);
}
__DEVICE__ double fabs(double __a)
{
return __nv_fabs(__a);
}
__DEVICE__ double acos(double __a)
{
return __nv_acos(__a);
}
__DEVICE__ float acosf(float __a)
{
return __nv_acosf(__a);
}
__DEVICE__ double acosh(double __a)
{
return __nv_acosh(__a);
}
__DEVICE__ float acoshf(float __a)
{
return __nv_acoshf(__a);
}
__DEVICE__ double asin(double __a)
{
return __nv_asin(__a);
}
__DEVICE__ float asinf(float __a)
{
return __nv_asinf(__a);
}
__DEVICE__ double asinh(double __a)
{
return __nv_asinh(__a);
}
__DEVICE__ float asinhf(float __a)
{
return __nv_asinhf(__a);
}
__DEVICE__ double atan(double __a)
{
return __nv_atan(__a);
}
__DEVICE__ double atan2(double __a, double __b)
{
return __nv_atan2(__a, __b);
}
__DEVICE__ float atan2f(float __a, float __b)
{
return __nv_atan2f(__a, __b);
}
__DEVICE__ float atanf(float __a)
{
return __nv_atanf(__a);
}
__DEVICE__ double atanh(double __a)
{
return __nv_atanh(__a);
}
__DEVICE__ float atanhf(float __a)
{
return __nv_atanhf(__a);
}
__DEVICE__ double cbrt(double __a)
{
return __nv_cbrt(__a);
}
__DEVICE__ float cbrtf(float __a)
{
return __nv_cbrtf(__a);
}
__DEVICE__ double ceil(double __a)
{
return __nv_ceil(__a);
}
__DEVICE__ float ceilf(float __a)
{
return __nv_ceilf(__a);
}
__DEVICE__ double copysign(double __a, double __b)
{
return __nv_copysign(__a, __b);
}
__DEVICE__ float copysignf(float __a, float __b)
{
return __nv_copysignf(__a, __b);
}
__DEVICE__ double cos(double __a)
{
return __nv_cos(__a);
}
__DEVICE__ float cosf(float __a)
{
return __FAST_OR_SLOW(__nv_fast_cosf, __nv_cosf)(__a);
}
__DEVICE__ double cosh(double __a)
{
return __nv_cosh(__a);
}
__DEVICE__ float coshf(float __a)
{
return __nv_coshf(__a);
}
__DEVICE__ double cospi(double __a)
{
return __nv_cospi(__a);
}
__DEVICE__ float cospif(float __a)
{
return __nv_cospif(__a);
}
__DEVICE__ double cyl_bessel_i0(double __a)
{
return __nv_cyl_bessel_i0(__a);
}
__DEVICE__ float cyl_bessel_i0f(float __a)
{
return __nv_cyl_bessel_i0f(__a);
}
__DEVICE__ double cyl_bessel_i1(double __a)
{
return __nv_cyl_bessel_i1(__a);
}
__DEVICE__ float cyl_bessel_i1f(float __a)
{
return __nv_cyl_bessel_i1f(__a);
}
__DEVICE__ double erf(double __a)
{
return __nv_erf(__a);
}
__DEVICE__ double erfc(double __a)
{
return __nv_erfc(__a);
}
__DEVICE__ float erfcf(float __a)
{
return __nv_erfcf(__a);
}
__DEVICE__ double erfcinv(double __a)
{
return __nv_erfcinv(__a);
}
__DEVICE__ float erfcinvf(float __a)
{
return __nv_erfcinvf(__a);
}
__DEVICE__ double erfcx(double __a)
{
return __nv_erfcx(__a);
}
__DEVICE__ float erfcxf(float __a)
{
return __nv_erfcxf(__a);
}
__DEVICE__ float erff(float __a)
{
return __nv_erff(__a);
}
__DEVICE__ double erfinv(double __a)
{
return __nv_erfinv(__a);
}
__DEVICE__ float erfinvf(float __a)
{
return __nv_erfinvf(__a);
}
__DEVICE__ double exp(double __a)
{
return __nv_exp(__a);
}
__DEVICE__ double exp10(double __a)
{
return __nv_exp10(__a);
}
__DEVICE__ float exp10f(float __a)
{
return __nv_exp10f(__a);
}
__DEVICE__ double exp2(double __a)
{
return __nv_exp2(__a);
}
__DEVICE__ float exp2f(float __a)
{
return __nv_exp2f(__a);
}
__DEVICE__ float expf(float __a)
{
return __nv_expf(__a);
}
__DEVICE__ double expm1(double __a)
{
return __nv_expm1(__a);
}
__DEVICE__ float expm1f(float __a)
{
return __nv_expm1f(__a);
}
__DEVICE__ float fabsf(float __a)
{
return __nv_fabsf(__a);
}
__DEVICE__ double fdim(double __a, double __b)
{
return __nv_fdim(__a, __b);
}
__DEVICE__ float fdimf(float __a, float __b)
{
return __nv_fdimf(__a, __b);
}
__DEVICE__ double fdivide(double __a, double __b)
{
return __a / __b;
}
__DEVICE__ float fdividef(float __a, float __b)
{
# if __FAST_MATH__ && !__CUDA_PREC_DIV
return __nv_fast_fdividef(__a, __b);
# else
return __a / __b;
# endif
}
__DEVICE__ double floor(double __f)
{
return __nv_floor(__f);
}
__DEVICE__ float floorf(float __f)
{
return __nv_floorf(__f);
}
__DEVICE__ double fma(double __a, double __b, double __c)
{
return __nv_fma(__a, __b, __c);
}
__DEVICE__ float fmaf(float __a, float __b, float __c)
{
return __nv_fmaf(__a, __b, __c);
}
__DEVICE__ double fmax(double __a, double __b)
{
return __nv_fmax(__a, __b);
}
__DEVICE__ float fmaxf(float __a, float __b)
{
return __nv_fmaxf(__a, __b);
}
__DEVICE__ double fmin(double __a, double __b)
{
return __nv_fmin(__a, __b);
}
__DEVICE__ float fminf(float __a, float __b)
{
return __nv_fminf(__a, __b);
}
__DEVICE__ double fmod(double __a, double __b)
{
return __nv_fmod(__a, __b);
}
__DEVICE__ float fmodf(float __a, float __b)
{
return __nv_fmodf(__a, __b);
}
__DEVICE__ double frexp(double __a, int* __b)
{
return __nv_frexp(__a, __b);
}
__DEVICE__ float frexpf(float __a, int* __b)
{
return __nv_frexpf(__a, __b);
}
__DEVICE__ double hypot(double __a, double __b)
{
return __nv_hypot(__a, __b);
}
__DEVICE__ float hypotf(float __a, float __b)
{
return __nv_hypotf(__a, __b);
}
__DEVICE__ int ilogb(double __a)
{
return __nv_ilogb(__a);
}
__DEVICE__ int ilogbf(float __a)
{
return __nv_ilogbf(__a);
}
__DEVICE__ double j0(double __a)
{
return __nv_j0(__a);
}
__DEVICE__ float j0f(float __a)
{
return __nv_j0f(__a);
}
__DEVICE__ double j1(double __a)
{
return __nv_j1(__a);
}
__DEVICE__ float j1f(float __a)
{
return __nv_j1f(__a);
}
__DEVICE__ double jn(int __n, double __a)
{
return __nv_jn(__n, __a);
}
__DEVICE__ float jnf(int __n, float __a)
{
return __nv_jnf(__n, __a);
}
# if defined(__LP64__) || defined(_WIN64)
__DEVICE__ long labs(long __a)
{
return __nv_llabs(__a);
};
# else
__DEVICE__ long labs(long __a)
{
return __nv_abs(__a);
};
# endif
__DEVICE__ double ldexp(double __a, int __b)
{
return __nv_ldexp(__a, __b);
}
__DEVICE__ float ldexpf(float __a, int __b)
{
return __nv_ldexpf(__a, __b);
}
__DEVICE__ double lgamma(double __a)
{
return __nv_lgamma(__a);
}
__DEVICE__ float lgammaf(float __a)
{
return __nv_lgammaf(__a);
}
__DEVICE__ long long llabs(long long __a)
{
return __nv_llabs(__a);
}
__DEVICE__ long long llmax(long long __a, long long __b)
{
return __nv_llmax(__a, __b);
}
__DEVICE__ long long llmin(long long __a, long long __b)
{
return __nv_llmin(__a, __b);
}
__DEVICE__ long long llrint(double __a)
{
return __nv_llrint(__a);
}
__DEVICE__ long long llrintf(float __a)
{
return __nv_llrintf(__a);
}
__DEVICE__ long long llround(double __a)
{
return __nv_llround(__a);
}
__DEVICE__ long long llroundf(float __a)
{
return __nv_llroundf(__a);
}
__DEVICE__ double round(double __a)
{
return __nv_round(__a);
}
__DEVICE__ float roundf(float __a)
{
return __nv_roundf(__a);
}
__DEVICE__ double log(double __a)
{
return __nv_log(__a);
}
__DEVICE__ double log10(double __a)
{
return __nv_log10(__a);
}
__DEVICE__ float log10f(float __a)
{
return __nv_log10f(__a);
}
__DEVICE__ double log1p(double __a)
{
return __nv_log1p(__a);
}
__DEVICE__ float log1pf(float __a)
{
return __nv_log1pf(__a);
}
__DEVICE__ double log2(double __a)
{
return __nv_log2(__a);
}
__DEVICE__ float log2f(float __a)
{
return __FAST_OR_SLOW(__nv_fast_log2f, __nv_log2f)(__a);
}
__DEVICE__ double logb(double __a)
{
return __nv_logb(__a);
}
__DEVICE__ float logbf(float __a)
{
return __nv_logbf(__a);
}
__DEVICE__ float logf(float __a)
{
return __FAST_OR_SLOW(__nv_fast_logf, __nv_logf)(__a);
}
# if defined(__LP64__) || defined(_WIN64)
__DEVICE__ long lrint(double __a)
{
return llrint(__a);
}
__DEVICE__ long lrintf(float __a)
{
return __float2ll_rn(__a);
}
__DEVICE__ long lround(double __a)
{
return llround(__a);
}
__DEVICE__ long lroundf(float __a)
{
return llroundf(__a);
}
# else
__DEVICE__ long lrint(double __a)
{
return (long) rint(__a);
}
__DEVICE__ long lrintf(float __a)
{
return __float2int_rn(__a);
}
__DEVICE__ long lround(double __a)
{
return round(__a);
}
__DEVICE__ long lroundf(float __a)
{
return roundf(__a);
}
# endif
__DEVICE__ int max(int __a, int __b)
{
return __nv_max(__a, __b);
}
__DEVICE__ int min(int __a, int __b)
{
return __nv_min(__a, __b);
}
__DEVICE__ double modf(double __a, double* __b)
{
return __nv_modf(__a, __b);
}
__DEVICE__ float modff(float __a, float* __b)
{
return __nv_modff(__a, __b);
}
__DEVICE__ double nearbyint(double __a)
{
return __builtin_nearbyint(__a);
}
__DEVICE__ float nearbyintf(float __a)
{
return __builtin_nearbyintf(__a);
}
__DEVICE__ double nextafter(double __a, double __b)
{
return __nv_nextafter(__a, __b);
}
__DEVICE__ float nextafterf(float __a, float __b)
{
return __nv_nextafterf(__a, __b);
}
__DEVICE__ double norm(int __dim, const double* __t)
{
return __nv_norm(__dim, __t);
}
__DEVICE__ double norm3d(double __a, double __b, double __c)
{
return __nv_norm3d(__a, __b, __c);
}
__DEVICE__ float norm3df(float __a, float __b, float __c)
{
return __nv_norm3df(__a, __b, __c);
}
__DEVICE__ double norm4d(double __a, double __b, double __c, double __d)
{
return __nv_norm4d(__a, __b, __c, __d);
}
__DEVICE__ float norm4df(float __a, float __b, float __c, float __d)
{
return __nv_norm4df(__a, __b, __c, __d);
}
__DEVICE__ double normcdf(double __a)
{
return __nv_normcdf(__a);
}
__DEVICE__ float normcdff(float __a)
{
return __nv_normcdff(__a);
}
__DEVICE__ double normcdfinv(double __a)
{
return __nv_normcdfinv(__a);
}
__DEVICE__ float normcdfinvf(float __a)
{
return __nv_normcdfinvf(__a);
}
__DEVICE__ float normf(int __dim, const float* __t)
{
return __nv_normf(__dim, __t);
}
__DEVICE__ double pow(double __a, double __b)
{
return __nv_pow(__a, __b);
}
__DEVICE__ float powf(float __a, float __b)
{
return __nv_powf(__a, __b);
}
__DEVICE__ double powi(double __a, int __b)
{
return __nv_powi(__a, __b);
}
__DEVICE__ float powif(float __a, int __b)
{
return __nv_powif(__a, __b);
}
__DEVICE__ double rcbrt(double __a)
{
return __nv_rcbrt(__a);
}
__DEVICE__ float rcbrtf(float __a)
{
return __nv_rcbrtf(__a);
}
__DEVICE__ double remainder(double __a, double __b)
{
return __nv_remainder(__a, __b);
}
__DEVICE__ float remainderf(float __a, float __b)
{
return __nv_remainderf(__a, __b);
}
__DEVICE__ double remquo(double __a, double __b, int* __c)
{
return __nv_remquo(__a, __b, __c);
}
__DEVICE__ float remquof(float __a, float __b, int* __c)
{
return __nv_remquof(__a, __b, __c);
}
__DEVICE__ double rhypot(double __a, double __b)
{
return __nv_rhypot(__a, __b);
}
__DEVICE__ float rhypotf(float __a, float __b)
{
return __nv_rhypotf(__a, __b);
}
// __nv_rint* in libdevice is buggy and produces incorrect results.
__DEVICE__ double rint(double __a)
{
return __builtin_rint(__a);
}
__DEVICE__ float rintf(float __a)
{
return __builtin_rintf(__a);
}
__DEVICE__ double rnorm(int __a, const double* __b)
{
return __nv_rnorm(__a, __b);
}
__DEVICE__ double rnorm3d(double __a, double __b, double __c)
{
return __nv_rnorm3d(__a, __b, __c);
}
__DEVICE__ float rnorm3df(float __a, float __b, float __c)
{
return __nv_rnorm3df(__a, __b, __c);
}
__DEVICE__ double rnorm4d(double __a, double __b, double __c, double __d)
{
return __nv_rnorm4d(__a, __b, __c, __d);
}
__DEVICE__ float rnorm4df(float __a, float __b, float __c, float __d)
{
return __nv_rnorm4df(__a, __b, __c, __d);
}
__DEVICE__ float rnormf(int __dim, const float* __t)
{
return __nv_rnormf(__dim, __t);
}
__DEVICE__ double rsqrt(double __a)
{
return __nv_rsqrt(__a);
}
__DEVICE__ float rsqrtf(float __a)
{
return __nv_rsqrtf(__a);
}
__DEVICE__ double scalbn(double __a, int __b)
{
return __nv_scalbn(__a, __b);
}
__DEVICE__ float scalbnf(float __a, int __b)
{
return __nv_scalbnf(__a, __b);
}
__DEVICE__ double scalbln(double __a, long __b)
{
if (__b > INT_MAX)
{
return __a > 0 ? HUGE_VAL : -HUGE_VAL;
}
if (__b < INT_MIN)
{
return __a > 0 ? 0.0 : -0.0;
}
return scalbn(__a, (int) __b);
}
__DEVICE__ float scalblnf(float __a, long __b)
{
if (__b > INT_MAX)
{
return __a > 0 ? HUGE_VALF : -HUGE_VALF;
}
if (__b < INT_MIN)
{
return __a > 0 ? 0.f : -0.f;
}
return scalbnf(__a, (int) __b);
}
__DEVICE__ double sin(double __a)
{
return __nv_sin(__a);
}
__DEVICE_VOID__ void sincos(double __a, double* __s, double* __c)
{
return __nv_sincos(__a, __s, __c);
}
__DEVICE_VOID__ void sincosf(float __a, float* __s, float* __c)
{
return __FAST_OR_SLOW(__nv_fast_sincosf, __nv_sincosf)(__a, __s, __c);
}
__DEVICE_VOID__ void sincospi(double __a, double* __s, double* __c)
{
return __nv_sincospi(__a, __s, __c);
}
__DEVICE_VOID__ void sincospif(float __a, float* __s, float* __c)
{
return __nv_sincospif(__a, __s, __c);
}
__DEVICE__ float sinf(float __a)
{
return __FAST_OR_SLOW(__nv_fast_sinf, __nv_sinf)(__a);
}
__DEVICE__ double sinh(double __a)
{
return __nv_sinh(__a);
}
__DEVICE__ float sinhf(float __a)
{
return __nv_sinhf(__a);
}
__DEVICE__ double sinpi(double __a)
{
return __nv_sinpi(__a);
}
__DEVICE__ float sinpif(float __a)
{
return __nv_sinpif(__a);
}
__DEVICE__ double sqrt(double __a)
{
return __nv_sqrt(__a);
}
__DEVICE__ float sqrtf(float __a)
{
return __nv_sqrtf(__a);
}
__DEVICE__ double tan(double __a)
{
return __nv_tan(__a);
}
__DEVICE__ float tanf(float __a)
{
return __nv_tanf(__a);
}
__DEVICE__ double tanh(double __a)
{
return __nv_tanh(__a);
}
__DEVICE__ float tanhf(float __a)
{
return __nv_tanhf(__a);
}
__DEVICE__ double tgamma(double __a)
{
return __nv_tgamma(__a);
}
__DEVICE__ float tgammaf(float __a)
{
return __nv_tgammaf(__a);
}
__DEVICE__ double trunc(double __a)
{
return __nv_trunc(__a);
}
__DEVICE__ float truncf(float __a)
{
return __nv_truncf(__a);
}
__DEVICE__ unsigned long long ullmax(unsigned long long __a, unsigned long long __b)
{
return __nv_ullmax(__a, __b);
}
__DEVICE__ unsigned long long ullmin(unsigned long long __a, unsigned long long __b)
{
return __nv_ullmin(__a, __b);
}
__DEVICE__ unsigned int umax(unsigned int __a, unsigned int __b)
{
return __nv_umax(__a, __b);
}
__DEVICE__ unsigned int umin(unsigned int __a, unsigned int __b)
{
return __nv_umin(__a, __b);
}
__DEVICE__ double y0(double __a)
{
return __nv_y0(__a);
}
__DEVICE__ float y0f(float __a)
{
return __nv_y0f(__a);
}
__DEVICE__ double y1(double __a)
{
return __nv_y1(__a);
}
__DEVICE__ float y1f(float __a)
{
return __nv_y1f(__a);
}
__DEVICE__ double yn(int __a, double __b)
{
return __nv_yn(__a, __b);
}
__DEVICE__ float ynf(int __a, float __b)
{
return __nv_ynf(__a, __b);
}
# pragma pop_macro("__DEVICE__")
# pragma pop_macro("__DEVICE_VOID__")
# pragma pop_macro("__FAST_OR_SLOW")
#endif // __CLANG_GPU_DISABLE_MATH_WRAPPERS
#endif // __CLANG_CUDA_MATH_H__

438
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/__clang_cuda_runtime_wrapper.h Normal file
View File

@@ -0,0 +1,438 @@
/*===---- HostJIT CUDA runtime wrapper - replaces clang's wrapper ----------===
*
* This is a self-contained replacement for clang's __clang_cuda_runtime_wrapper.h.
* Instead of #include_next-ing the real wrapper (which has fragile ordering
* dependencies on system headers and CUDA toolkit version-specific branches),
* we directly include only the clang-provided CUDA helper headers we need and
* pull in the CUDA toolkit headers with explicit preprocessor guards.
*
* Key design decision: all clang-provided device function implementations and
* CCCL-required intrinsics are defined BEFORE any CUDA toolkit headers that
* might transitively include CCCL (via libcudacxx standard headers on our
* include path). This eliminates the need for forward declarations.
*
* Assumptions:
* - CUDA >= 9.0 (no legacy code paths)
* - Clang CUDA compilation (__CUDA__ && __clang__)
* - Freestanding: all standard headers are stubs or from libcudacxx
* - cuda::std is bridged into std via using-directive
*===-----------------------------------------------------------------------===*/
#ifndef __CLANG_CUDA_RUNTIME_WRAPPER_H__
#define __CLANG_CUDA_RUNTIME_WRAPPER_H__
#pragma clang system_header
#if defined(__CUDA__) && defined(__clang__)
// ============================================================================
// Phase 1: Forward-declare device math overloads before any <cmath> inclusion
// ============================================================================
// This prevents constexpr std library math functions from becoming implicitly
// host+device, which would block our __device__ overloads later.
# include <__clang_cuda_math_forward_declares.h>
// ============================================================================
// Phase 2: Device-side definitions before any CUDA toolkit headers
// ============================================================================
// Everything here uses only compiler builtins and our stubs. No CUDA toolkit
// headers are included yet, so nothing can transitively pull in CCCL.
# pragma push_macro("__THROW")
# pragma push_macro("__CUDA_ARCH__")
# ifndef __CUDA_ARCH__
# define __CUDA_ARCH__ 9999
# endif
// host_defines.h provides __device__, __host__, __forceinline__ macros.
// Its only transitive dep (ctype.h) hits our stub.
# define __CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__
# define __CUDACC__
# define __CUDA_LIBDEVICE__
# include "host_defines.h"
// ---- Builtin variables (threadIdx, blockIdx, etc.) ----
# include "__clang_cuda_builtin_vars.h"
// ---- Stubs needed by clang device function headers below ----
# include <climits>
# include <cmath>
# include <cstddef>
// string.h must precede __clang_cuda_device_functions.h: cuda_fp16.hpp uses
// memcpy from __host__ __device__ ctors. device_functions.h only declares a
// __device__ memcpy, so the host-side call site needs the stub's host-callable
// __builtin_memcpy overload visible first.
# include <string.h>
// ---- Clang device function wrappers (local copies, CUDA < 9.0 removed) ----
// NOTE: libdevice_declares.h must precede device_functions.h — the latter calls
// __nv_* symbols that are declared in the former.
// clang-format off
# include "__clang_cuda_libdevice_declares.h"
# include "__clang_cuda_device_functions.h"
// clang-format on
# include "__clang_cuda_math.h"
// ---- Address-space intrinsics needed by CCCL headers ----
// (e.g. cuda/__memory/address_space.h, cuda/__ptx/ptx_helper_functions.h)
static __device__ __forceinline__ __attribute__((const)) unsigned int __isGlobal(const void* p)
{
return __nvvm_isspacep_global(p);
}
static __device__ __forceinline__ __attribute__((const)) unsigned int __isShared(const void* p)
{
return __nvvm_isspacep_shared(p);
}
static __device__ __forceinline__ __attribute__((const)) unsigned int __isConstant(const void* p)
{
return __nvvm_isspacep_const(p);
}
static __device__ __forceinline__ __attribute__((const)) unsigned int __isLocal(const void* p)
{
return __nvvm_isspacep_local(p);
}
# define __FWD_DEVICE static __device__ __forceinline__
__FWD_DEVICE unsigned int __isClusterShared(const void*);
__FWD_DEVICE __SIZE_TYPE__ __cvta_generic_to_shared(const void*);
__FWD_DEVICE __SIZE_TYPE__ __cvta_generic_to_global(const void*);
__FWD_DEVICE void* __cvta_shared_to_generic(__SIZE_TYPE__);
__FWD_DEVICE void* __cvta_global_to_generic(__SIZE_TYPE__);
# undef __FWD_DEVICE
# ifndef _MSC_VER
__device__ bool __nv_fp128_isnan(__float128);
__device__ __float128 __nv_fp128_fmax(__float128, __float128);
__device__ __float128 __nv_fp128_fmin(__float128, __float128);
# endif
// ---- Bridge cuda::std into std ----
namespace cuda
{
namespace std
{
}
} // namespace cuda
namespace std
{
using namespace cuda::std;
}
// ============================================================================
// Phase 3: CUDA toolkit headers
// ============================================================================
// By this point all device-side functions and intrinsics are defined, so
// any transitive CCCL includes from these headers will find them.
# pragma push_macro("__CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__")
# define __DEVICE_LAUNCH_PARAMETERS_H__
// Guard out CUDA's declaration-only headers; clang provides its own.
# define __DEVICE_FUNCTIONS_H__
# define __MATH_FUNCTIONS_H__
# define __MATH_FUNCTIONS_HPP__
# define __COMMON_FUNCTIONS_H__
# define __DEVICE_FUNCTIONS_DECLS_H__
// ---- CUDA runtime types (cudaError_t, dim3, cudaStream_t, etc.) ----
// (host_defines.h already included in Phase 2)
# undef __CUDACC__
# include "cuda.h"
# include "driver_types.h"
# include "host_config.h"
# if !defined(CUDA_VERSION) || CUDA_VERSION < 9000
# error "Unsupported CUDA version (need >= 9.0)!"
# endif
// Clang does not have __nvvm_memcpy/__nvvm_memset; emulate with builtins.
# define __nvvm_memcpy(s, d, n, a) __builtin_memcpy(s, d, n)
# define __nvvm_memset(d, c, n, a) __builtin_memset(d, c, n)
// __THROW may be in a weird state; keep it empty for CUDA includes.
# undef __THROW
# define __THROW
// ============================================================================
// Phase 4: Device-side function definitions from CUDA toolkit .hpp files
// ============================================================================
// Poison __host__ to ensure none of these definitions get host attributes.
# pragma push_macro("__host__")
# define __host__ UNEXPECTED_HOST_ATTRIBUTE
// Redefine __forceinline__ to include __device__.
# pragma push_macro("__forceinline__")
# define __forceinline__ __device__ __inline__ __attribute__((always_inline))
// Math functions: use fast or accurate variants based on compiler flag.
# pragma push_macro("__USE_FAST_MATH__")
# if defined(__CLANG_GPU_APPROX_TRANSCENDENTALS__)
# define __USE_FAST_MATH__ 1
# endif
# include "crt/math_functions.hpp"
# pragma pop_macro("__USE_FAST_MATH__")
# pragma pop_macro("__forceinline__")
# undef __MATH_FUNCTIONS_HPP__
# undef __CUDABE__
// Re-include device functions with __host__ defined as empty to get
// the "other branch" of #if/#else in the .hpp files.
# define __host__
# undef __CUDABE__
# define __CUDACC__
// Atomic function declarations (became builtins in CUDA 9).
# include "device_atomic_functions.h"
# undef __DEVICE_FUNCTIONS_HPP__
# include "crt/device_double_functions.hpp"
# include "crt/device_functions.hpp"
# include "device_atomic_functions.hpp"
# include "sm_20_atomic_functions.hpp"
// sm_20_intrinsics.hpp defines __isGlobal etc. without const attribute.
// Rename them so the definitions from Phase 4 (with const) prevail.
# pragma push_macro("__isGlobal")
# pragma push_macro("__isShared")
# pragma push_macro("__isConstant")
# pragma push_macro("__isLocal")
# define __isGlobal __ignored_cuda___isGlobal
# define __isShared __ignored_cuda___isShared
# define __isConstant __ignored_cuda___isConstant
# define __isLocal __ignored_cuda___isLocal
# include "sm_20_intrinsics.hpp"
# pragma pop_macro("__isGlobal")
# pragma pop_macro("__isShared")
# pragma pop_macro("__isConstant")
# pragma pop_macro("__isLocal")
# include "sm_32_atomic_functions.hpp"
# pragma push_macro("__CUDA_ARCH__")
# undef __CUDA_ARCH__
# include "sm_60_atomic_functions.hpp"
# include "sm_61_intrinsics.hpp"
# pragma pop_macro("__CUDA_ARCH__")
# undef __MATH_FUNCTIONS_HPP__
// math_functions.hpp ::signbit conflicts with libstdc++ constexpr ::signbit.
# pragma push_macro("signbit")
# pragma push_macro("__GNUC__")
# undef __GNUC__
# define signbit __ignored_cuda_signbit
# pragma push_macro("_GLIBCXX_MATH_H")
# pragma push_macro("_LIBCPP_VERSION")
# undef _GLIBCXX_MATH_H
# ifdef _LIBCPP_VERSION
# define _LIBCPP_VERSION 3700
# endif
# include "crt/math_functions.hpp"
# pragma pop_macro("_GLIBCXX_MATH_H")
# pragma pop_macro("_LIBCPP_VERSION")
# pragma pop_macro("__GNUC__")
# pragma pop_macro("signbit")
# pragma pop_macro("__host__")
// ============================================================================
// Phase 5: cuda_runtime.h (first header that transitively pulls in CCCL)
// ============================================================================
// ============================================================================
// Phase 5: cuda_runtime.h (first header that transitively pulls in CCCL)
// ============================================================================
// Verify no libcudacxx header was pulled in yet. If this fires, a header
// above transitively included a system header that resolved to libcudacxx
// before all device-side definitions were ready.
# ifdef CCCL_VERSION
# error "libcudacxx was included before device-side definitions were set up"
# endif
# pragma push_macro("nv_weak")
# define nv_weak weak
# undef __CUDA_LIBDEVICE__
# define __CUDACC__
# include "cuda_runtime.h"
# pragma pop_macro("nv_weak")
# undef __CUDACC__
# define __CUDABE__
# include "crt/host_runtime.h"
// device_runtime.h defines __cxa_* macros that conflict with cxxabi.h.
# undef __cxa_vec_ctor
# undef __cxa_vec_cctor
# undef __cxa_vec_dtor
# undef __cxa_vec_new
# undef __cxa_vec_new2
# undef __cxa_vec_new3
# undef __cxa_vec_delete2
# undef __cxa_vec_delete
# undef __cxa_vec_delete3
# undef __cxa_pure_virtual
// Texture intrinsics (requires C++11).
# if __cplusplus >= 201103L
# include <__clang_cuda_texture_intrinsics.h>
# else
template <typename T>
struct __nv_tex_needs_cxx11
{
const static bool value = false;
};
template <class T>
__host__ __device__ void __nv_tex_surf_handler(const char* name, T* ptr, cudaTextureObject_t obj, float x)
{
_Static_assert(__nv_tex_needs_cxx11<T>::value, "Texture support requires C++11");
}
# endif
# include "surface_indirect_functions.h"
# if CUDA_VERSION < 13000
# include "texture_fetch_functions.h"
# endif
# include "texture_indirect_functions.h"
// ============================================================================
// Phase 7: Restore saved state
// ============================================================================
# pragma pop_macro("__CUDA_ARCH__")
# pragma pop_macro("__THROW")
# undef __CUDABE__
# define __CUDACC__
// ============================================================================
// Phase 8: Device-side system calls & std wrappers
// ============================================================================
extern "C" {
__device__ int vprintf(const char*, const char*);
__device__ void free(void*) __attribute((nothrow));
__device__ void* malloc(size_t) __attribute((nothrow)) __attribute__((malloc));
__device__ void
__assertfail(const char* __message, const char* __file, unsigned __line, const char* __function, size_t __charSize);
__device__ static inline void
__assert_fail(const char* __message, const char* __file, unsigned __line, const char* __function)
{
__assertfail(__message, __file, __line, __function, sizeof(char));
}
__device__ int printf(const char*, ...);
} // extern "C"
namespace std
{
__device__ static inline void free(void* __ptr)
{
::free(__ptr);
}
__device__ static inline void* malloc(size_t __size)
{
return ::malloc(__size);
}
} // namespace std
// ============================================================================
// Phase 9: Builtin variable conversion operators
// ============================================================================
// These need dim3 and uint3 to be fully defined (from vector_types.h, pulled
// in by driver_types.h in Phase 5).
__device__ inline __cuda_builtin_threadIdx_t::operator dim3() const
{
return dim3(x, y, z);
}
__device__ inline __cuda_builtin_threadIdx_t::operator uint3() const
{
return {x, y, z};
}
__device__ inline __cuda_builtin_blockIdx_t::operator dim3() const
{
return dim3(x, y, z);
}
__device__ inline __cuda_builtin_blockIdx_t::operator uint3() const
{
return {x, y, z};
}
__device__ inline __cuda_builtin_blockDim_t::operator dim3() const
{
return dim3(x, y, z);
}
__device__ inline __cuda_builtin_blockDim_t::operator uint3() const
{
return {x, y, z};
}
__device__ inline __cuda_builtin_gridDim_t::operator dim3() const
{
return dim3(x, y, z);
}
__device__ inline __cuda_builtin_gridDim_t::operator uint3() const
{
return {x, y, z};
}
// ============================================================================
// Phase 10: Remaining clang CUDA headers
// ============================================================================
# include <__clang_cuda_cmath.h>
# include <__clang_cuda_intrinsics.h>
// __clang_cuda_intrinsics.h provides `long` overloads for __ldcs/__ldcg/__ldcv
// but omits `unsigned long` (= uint64_t on 64-bit Linux). Add them here so
// iterators using uint64_t pointers (e.g. CacheModifiedInputIterator) compile.
# if defined(__LP64__)
inline __device__ unsigned long __ldcs(const unsigned long* __ptr)
{
unsigned long __ret;
asm("ld.global.cs.u64 %0, [%1];" : "=l"(__ret) : "l"(__ptr));
return __ret;
}
inline __device__ unsigned long __ldcg(const unsigned long* __ptr)
{
unsigned long __ret;
asm("ld.global.cg.u64 %0, [%1];" : "=l"(__ret) : "l"(__ptr));
return __ret;
}
inline __device__ unsigned long __ldcv(const unsigned long* __ptr)
{
unsigned long __ret;
asm("ld.global.cv.u64 %0, [%1];" : "=l"(__ret) : "l"(__ptr));
return __ret;
}
# endif // __LP64__
# include <__clang_cuda_complex_builtins.h>
// curand_mtgp32_kernel redefines blockDim/threadIdx with dim3/uint3 types,
// which is incompatible with our builtins. Force-include it with types
// redefined to our builtin types.
// Skip when cuRAND headers are unavailable (e.g. pip-installed toolkit).
# if __has_include("curand_mtgp32_kernel.h")
# pragma push_macro("dim3")
# pragma push_macro("uint3")
# define dim3 __cuda_builtin_blockDim_t
# define uint3 __cuda_builtin_threadIdx_t
# include "curand_mtgp32_kernel.h"
# pragma pop_macro("dim3")
# pragma pop_macro("uint3")
# endif
# pragma pop_macro("__CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__")
// Kernel launch configuration function.
# if CUDA_VERSION >= 9020
extern "C" unsigned __cudaPushCallConfiguration(dim3 gridDim, dim3 blockDim, size_t sharedMem = 0, void* stream = 0);
# endif
// The JIT shared library is linked without the C runtime (no libc on the link
// line) so atexit is unavailable. The CUDA module constructor calls atexit()
// to register a cleanup function. Provide a no-op stub — the JIT library is
// short-lived and unloaded explicitly.
# if !defined(__HOSTJIT_DEVICE_COMPILATION__)
# if defined(_MSC_VER)
extern "C" int atexit(void(__cdecl*)(void))
{
return 0;
}
# else
extern "C" int atexit(void (*)(void))
{
return 0;
}
# endif
# endif
#endif // __CUDA__ && __clang__
#endif // __CLANG_CUDA_RUNTIME_WRAPPER_H__

31
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/assert.h Normal file
View File

@@ -0,0 +1,31 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
// Minimal freestanding-mode stub for <assert.h>.
//
// CUDA toolkit headers pulled in via libcudacxx's __floating_point/cuda_fp_types.h
// (e.g. cuda_fp8.hpp) include <assert.h> unconditionally. In the JIT compile
// environment we have no libc; treat assert(expr) as a no-op. This matches the
// effect of `-DNDEBUG`, which CCCL/CUB device code already expects.
#ifndef _HOSTJIT_ASSERT_H
#define _HOSTJIT_ASSERT_H
#ifdef __cplusplus
extern "C" {
#endif
#undef assert
#define assert(expr) ((void) 0)
#ifdef __cplusplus
}
#endif
#endif // _HOSTJIT_ASSERT_H

6
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/cassert Normal file
View File

@@ -0,0 +1,6 @@
// Minimal freestanding-mode stub for <cassert>.
// Just delegate to <assert.h>'s no-op assert.
#ifndef _HOSTJIT_CASSERT
#define _HOSTJIT_CASSERT
#include <assert.h>
#endif // _HOSTJIT_CASSERT

7
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/climits Normal file
View File

@@ -0,0 +1,7 @@
// Minimal climits stub for CUDA JIT compilation
#ifndef _HOSTJIT_CLIMITS
#define _HOSTJIT_CLIMITS
#include <limits.h>
#endif // _HOSTJIT_CLIMITS

7
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/cmath Normal file
View File

@@ -0,0 +1,7 @@
// Minimal cmath stub for CUDA JIT compilation
#ifndef _HOSTJIT_CMATH
#define _HOSTJIT_CMATH
#include <math.h>
#endif // _HOSTJIT_CMATH

15
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/cstddef Normal file
View File

@@ -0,0 +1,15 @@
// Minimal cstddef stub for CUDA JIT compilation
// Compatible with libcu++ which expects to pull types from global namespace
#ifndef _HOSTJIT_CSTDDEF
#define _HOSTJIT_CSTDDEF
#include <stddef.h>
namespace std {
using ::size_t;
using ::ptrdiff_t;
using nullptr_t = decltype(nullptr);
}
#endif // _HOSTJIT_CSTDDEF

12
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/cstdlib Normal file
View File

@@ -0,0 +1,12 @@
#ifndef _HOSTJIT_CSTDLIB
#define _HOSTJIT_CSTDLIB
#include <cstddef>
#define EXIT_SUCCESS 0
#define EXIT_FAILURE 1
#define RAND_MAX 2147483647
extern "C" {
void* malloc(size_t); void* calloc(size_t, size_t);
void* realloc(void*, size_t); void free(void*);
void abort(void); void exit(int); void _Exit(int);
}
#endif

3
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/ctype.h Normal file
View File

@@ -0,0 +1,3 @@
#ifndef _HOSTJIT_CTYPE_H
#define _HOSTJIT_CTYPE_H
#endif

75
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/cuda/std/__cstdlib/aligned_alloc.h Normal file
View File

@@ -0,0 +1,75 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
// ClangJIT minimal stub for cuda/std/__cstdlib/aligned_alloc.h
//
// Problem: hostjit compiles with _CCCL_ENABLE_FREESTANDING=1 in both device
// and host passes. The host pass needs ::cuda::std::__aligned_alloc_host, but
// the real header gates that function on _CCCL_HOSTED(), which is 0 in a
// freestanding build.
//
// Solution: replace the entire header with a bare-metal stub that uses only
// compiler builtins (__builtin_malloc, __SIZE_TYPE__) and NO CCCL headers.
// Including CCCL headers from within this stub caused __clang_cuda_device_functions.h
// to be re-processed before __clang_cuda_libdevice_declares.h during device
// compilation, producing "undeclared identifier __nv_ull2float_rz" errors.
//
// __builtin_malloc is a compiler intrinsic — no headers required.
// __SIZE_TYPE__ is a compiler predefined macro equal to the platform size_t type.
//
// Neither path is ever actually called at runtime:
// - Host pass: CUB dispatch never calls aligned_alloc in our generated source.
// - Device pass: NV_IF_ELSE_TARGET discards the NV_IS_HOST branch at compile time.
#ifndef _CUDA_STD___CSTDLIB_ALIGNED_ALLOC_H
#define _CUDA_STD___CSTDLIB_ALIGNED_ALLOC_H
#if defined(__CUDA_ARCH__)
// ── Device compilation ────────────────────────────────────────────────────
// Provide cuda::std::aligned_alloc via the CUDA device syscall.
// The NV_IS_HOST branch of the CUB include chain is discarded by Clang's
// "if target" extension, so this function is never actually called.
extern "C" __device__ void* __cuda_syscall_aligned_malloc(__SIZE_TYPE__, __SIZE_TYPE__);
namespace cuda
{
namespace std
{
inline __device__ void* aligned_alloc(__SIZE_TYPE__ __align, __SIZE_TYPE__ __nbytes) noexcept
{
return ::__cuda_syscall_aligned_malloc(__nbytes, __align);
}
} // namespace std
} // namespace cuda
#else
// ── Host compilation ──────────────────────────────────────────────────────
// Define __aligned_alloc_host unconditionally so the CUB include chain
// compiles even when _CCCL_HOSTED() == 0. __builtin_malloc needs no headers.
namespace cuda
{
namespace std
{
inline void* __aligned_alloc_host(__SIZE_TYPE__, __SIZE_TYPE__ __nbytes) noexcept
{
return __builtin_malloc(__nbytes);
}
inline void* aligned_alloc(__SIZE_TYPE__ __align, __SIZE_TYPE__ __nbytes) noexcept
{
return ::cuda::std::__aligned_alloc_host(__align, __nbytes);
}
} // namespace std
} // namespace cuda
#endif // __CUDA_ARCH__
#endif // _CUDA_STD___CSTDLIB_ALIGNED_ALLOC_H

47
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/initializer_list Normal file
View File

@@ -0,0 +1,47 @@
// Minimal initializer_list stub for CUDA JIT compilation
#ifndef _HOSTJIT_INITIALIZER_LIST
#define _HOSTJIT_INITIALIZER_LIST
#include <cstddef>
namespace std {
template<class T>
class initializer_list {
public:
using value_type = T;
using reference = const T&;
using const_reference = const T&;
using size_type = size_t;
using iterator = const T*;
using const_iterator = const T*;
private:
const T* _begin;
size_t _size;
// This constructor is called by the compiler
constexpr initializer_list(const T* b, size_t s) noexcept
: _begin(b), _size(s) {}
public:
constexpr initializer_list() noexcept : _begin(nullptr), _size(0) {}
constexpr size_t size() const noexcept { return _size; }
constexpr const T* begin() const noexcept { return _begin; }
constexpr const T* end() const noexcept { return _begin + _size; }
};
template<class T>
constexpr const T* begin(initializer_list<T> il) noexcept {
return il.begin();
}
template<class T>
constexpr const T* end(initializer_list<T> il) noexcept {
return il.end();
}
} // namespace std
#endif // _HOSTJIT_INITIALIZER_LIST

61
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/limits Normal file
View File

@@ -0,0 +1,61 @@
// Minimal <limits> stub for hostjit device compilation.
//
// Clang's __clang_cuda_cmath.h includes <limits> unconditionally, then expands
// __CUDA_CLANG_FN_INTEGER_OVERLOAD_1/2 macros that reference
// std::numeric_limits<__T>::is_integer in return-type SFINAE at parse time.
// Clang evaluates these dependent names during template parsing, so the struct
// must be declared — not just forward-declared — before the macro expansion.
//
// In the hostjit device-compilation include path, <limits> would normally
// resolve to libcudacxx/include/cuda/std/limits, which cascades through
// numeric_limits, bit_cast, popcount, etc. — incompatible with freestanding.
//
// This stub (found first on -internal-isystem) stops that cascade, providing
// only the two members that __clang_cuda_cmath.h actually inspects.
#pragma once
namespace std {
template <typename _Tp>
struct numeric_limits {
static constexpr bool is_specialized = false;
static constexpr bool is_integer = false;
};
// Integer specializations — needed so the SFINAE in __clang_cuda_cmath.h
// correctly dispatches integer arguments.
#define _HOSTJIT_NUM_LIM_INT(_T) \
template <> struct numeric_limits<_T> { \
static constexpr bool is_specialized = true; \
static constexpr bool is_integer = true; \
};
_HOSTJIT_NUM_LIM_INT(bool)
_HOSTJIT_NUM_LIM_INT(char)
_HOSTJIT_NUM_LIM_INT(signed char)
_HOSTJIT_NUM_LIM_INT(unsigned char)
_HOSTJIT_NUM_LIM_INT(short)
_HOSTJIT_NUM_LIM_INT(unsigned short)
_HOSTJIT_NUM_LIM_INT(int)
_HOSTJIT_NUM_LIM_INT(unsigned int)
_HOSTJIT_NUM_LIM_INT(long)
_HOSTJIT_NUM_LIM_INT(unsigned long)
_HOSTJIT_NUM_LIM_INT(long long)
_HOSTJIT_NUM_LIM_INT(unsigned long long)
#undef _HOSTJIT_NUM_LIM_INT
// Floating-point specializations.
#define _HOSTJIT_NUM_LIM_FP(_T) \
template <> struct numeric_limits<_T> { \
static constexpr bool is_specialized = true; \
static constexpr bool is_integer = false; \
};
_HOSTJIT_NUM_LIM_FP(float)
_HOSTJIT_NUM_LIM_FP(double)
_HOSTJIT_NUM_LIM_FP(long double)
#undef _HOSTJIT_NUM_LIM_FP
} // namespace std

21
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/math.h Normal file
View File

@@ -0,0 +1,21 @@
#ifndef _HOSTJIT_MATH_H
#define _HOSTJIT_MATH_H
// Macros needed by __clang_cuda_math.h
#define HUGE_VAL __builtin_huge_val()
#define HUGE_VALF __builtin_huge_valf()
#define HUGE_VALL __builtin_huge_vall()
#define INFINITY __builtin_inff()
#define NAN __builtin_nanf("")
#define MATH_ERRNO 1
#define MATH_ERREXCEPT 2
#define math_errhandling (MATH_ERRNO | MATH_ERREXCEPT)
#define FP_NAN 0
#define FP_INFINITE 1
#define FP_ZERO 2
#define FP_SUBNORMAL 3
#define FP_NORMAL 4
#define __signbit(x) __builtin_signbit(x)
#define __signbitl(x) __builtin_signbitl(x)
#endif

4
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/memory.h Normal file
View File

@@ -0,0 +1,4 @@
#ifndef _HOSTJIT_MEMORY_H
#define _HOSTJIT_MEMORY_H
#include <string.h>
#endif

29
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/new Normal file
View File

@@ -0,0 +1,29 @@
#ifndef _HOSTJIT_NEW
#define _HOSTJIT_NEW
#include <cstddef>
namespace std {
struct nothrow_t { explicit nothrow_t() = default; };
extern const nothrow_t nothrow;
enum class align_val_t : size_t {};
}
// Placement new — needs __host__ __device__ for CUDA
#if defined(__CUDA__)
__host__ __device__
#endif
inline void* operator new(std::size_t, void* p) noexcept { return p; }
#if defined(__CUDA__)
__host__ __device__
#endif
inline void* operator new[](std::size_t, void* p) noexcept { return p; }
#if defined(__CUDA__)
__host__ __device__
#endif
inline void operator delete(void*, void*) noexcept {}
#if defined(__CUDA__)
__host__ __device__
#endif
inline void operator delete[](void*, void*) noexcept {}
#endif

11
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/stdlib.h Normal file
View File

@@ -0,0 +1,11 @@
// Minimal stdlib.h stub for CUDA JIT compilation
#ifndef _HOSTJIT_STDLIB_H
#define _HOSTJIT_STDLIB_H
#include <cstdlib>
#ifdef _WIN32
extern "C" int _fltused = 0;
#endif // _WIN32
#endif // _HOSTJIT_STDLIB_H

72
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/string.h Normal file
View File

@@ -0,0 +1,72 @@
#ifndef _HOSTJIT_STRING_H
#define _HOSTJIT_STRING_H
#include <stddef.h>
#ifdef __cplusplus
extern "C" {
inline void* memcpy(void* __s1, const void* __s2, size_t __n)
{
return __builtin_memcpy(__s1, __s2, __n);
}
inline void* memset(void* __s, int __c, size_t __n)
{
return __builtin_memset(__s, __c, __n);
}
inline void* memmove(void* __s1, const void* __s2, size_t __n)
{
return __builtin_memmove(__s1, __s2, __n);
}
inline int memcmp(const void* __s1, const void* __s2, size_t __n)
{
return __builtin_memcmp(__s1, __s2, __n);
}
inline char* strchr(char* __s, int __c)
{
return __builtin_strchr(__s, __c);
}
inline char* strpbrk(char* __s1, const char* __s2)
{
return __builtin_strpbrk(__s1, __s2);
}
inline char* strrchr(char* __s, int __c)
{
return __builtin_strrchr(__s, __c);
}
inline void* memchr(void* __s, int __c, size_t __n)
{
return __builtin_memchr(__s, __c, __n);
}
inline char* strstr(char* __s1, const char* __s2)
{
return __builtin_strstr(__s1, __s2);
}
inline char* strcpy(char* __s1, const char* __s2)
{
return __builtin_strcpy(__s1, __s2);
}
inline char* strncpy(char* __s1, const char* __s2, size_t __n)
{
return __builtin_strncpy(__s1, __s2, __n);
}
inline int strcmp(const char* __s1, const char* __s2)
{
return __builtin_strcmp(__s1, __s2);
}
inline int strncmp(const char* __s1, const char* __s2, size_t __n)
{
return __builtin_strncmp(__s1, __s2, __n);
}
inline size_t strlen(const char* __s)
{
return __builtin_strlen(__s);
}
}
#else // ^^^ __cplusplus ^^^ / vvv !__cplusplus vvv
void* memcpy(void*, const void*, size_t);
void* memset(void*, int, size_t);
int memcmp(const void*, const void*, size_t);
void* memmove(void*, const void*, size_t);
size_t strlen(const char*);
#endif // !__cplusplus
#endif //_HOSTJIT_STRING_H

34
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/cuda_minimal/stubs/utility Normal file
View File

@@ -0,0 +1,34 @@
// Minimal <utility> stub for hostjit device compilation.
//
// cuda_runtime.h includes <utility> for std::forward/std::move. In the
// hostjit device-compilation include path, <utility> resolves to
// libcudacxx/include/cuda/std/utility, which cascades into the full CCCL
// utility/iterator/concepts hierarchy — incompatible with freestanding mode.
//
// This stub (found first on -internal-isystem) stops that cascade.
// Only std::forward and std::move are provided because that is all
// cuda_runtime.h actually uses at the top level; the full CCCL hierarchy
// is not required for a simple host+device kernel.
#pragma once
namespace std {
template <typename _Tp> struct remove_reference { using type = _Tp; };
template <typename _Tp> struct remove_reference<_Tp&> { using type = _Tp; };
template <typename _Tp> struct remove_reference<_Tp&&>{ using type = _Tp; };
template <typename _Tp>
using remove_reference_t = typename remove_reference<_Tp>::type;
template <typename _Tp>
__host__ __device__ constexpr _Tp&&
forward(remove_reference_t<_Tp>& __t) noexcept { return static_cast<_Tp&&>(__t); }
template <typename _Tp>
__host__ __device__ constexpr _Tp&&
forward(remove_reference_t<_Tp>&& __t) noexcept { return static_cast<_Tp&&>(__t); }
template <typename _Tp>
__host__ __device__ constexpr remove_reference_t<_Tp>&&
move(_Tp&& __t) noexcept { return static_cast<remove_reference_t<_Tp>&&>(__t); }
} // namespace std

95
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/jit_compiler.hpp Normal file
View File

@@ -0,0 +1,95 @@
#pragma once
#include <memory>
#include <string>
#include <hostjit/compiler.hpp>
#include <hostjit/config.hpp>
#include <hostjit/loader.hpp>
namespace hostjit
{
class JITCompiler
{
public:
// Create JIT compiler with default configuration (auto-detected)
JITCompiler();
// Create JIT compiler with custom configuration
explicit JITCompiler(const CompilerConfig& config);
~JITCompiler();
// Disable copy
JITCompiler(const JITCompiler&) = delete;
JITCompiler& operator=(const JITCompiler&) = delete;
// Compile CUDA source code to shared library and load it
// Returns true on success, false on failure
bool compile(const std::string& source_code);
// Get function pointer by name
// Returns nullptr if function not found
template <typename FuncType>
FuncType getFunction(const std::string& name)
{
if (!library_.isLoaded())
{
last_error_ = "No library loaded";
return nullptr;
}
auto func = library_.getFunction<FuncType>(name);
if (!func)
{
last_error_ = "Failed to find function '" + name + "': " + library_.getLastError();
}
return func;
}
// Get the last error message
std::string getLastError() const
{
return last_error_;
}
// Get the configuration being used
const CompilerConfig& getConfig() const
{
return config_;
}
// Check if a library is currently loaded
bool isLoaded() const
{
return library_.isLoaded();
}
// Get the path to compiled artifacts (object file, shared library, etc.)
// Only valid after successful compile() and if keep_artifacts is set
std::string getArtifactsPath() const
{
return temp_dir_;
}
// Get the cubin extracted during compilation
const std::vector<char>& getCubin() const
{
return cubin_;
}
// Unload the current library and clean up temporary files
void cleanup();
private:
std::string createTempDirectory();
void removeTempDirectory();
CompilerConfig config_;
CUDACompiler compiler_;
DynamicLibrary library_;
std::string temp_dir_;
std::string last_error_;
std::vector<char> cubin_;
};
} // namespace hostjit

47
cccl_upstream/c/parallel.v2/src/hostjit/include/hostjit/loader.hpp Normal file
View File

@@ -0,0 +1,47 @@
#pragma once
#include <string>
namespace hostjit
{
class DynamicLibrary
{
public:
DynamicLibrary();
~DynamicLibrary();
// Disable copy
DynamicLibrary(const DynamicLibrary&) = delete;
DynamicLibrary& operator=(const DynamicLibrary&) = delete;
// Enable move
DynamicLibrary(DynamicLibrary&& other) noexcept;
DynamicLibrary& operator=(DynamicLibrary&& other) noexcept;
// Load a shared library
bool load(const std::string& library_path);
// Get a symbol (function or variable) by name
void* getSymbol(const std::string& symbol_name);
// Template helper to get function pointers with type safety
template <typename FuncType>
FuncType getFunction(const std::string& name)
{
return reinterpret_cast<FuncType>(getSymbol(name));
}
// Check if library is loaded
bool isLoaded() const;
// Get the last error message
std::string getLastError() const;
// Unload the library
void unload();
private:
void* handle_;
std::string last_error_;
};
} // namespace hostjit

192
cccl_upstream/c/parallel.v2/src/hostjit/jit_compiler.cpp Normal file
View File

@@ -0,0 +1,192 @@
#include <cstdlib>
#include <filesystem>
#include <iostream>
#include <random>
#include <sstream>
#include <hostjit/jit_compiler.hpp>
#ifdef _WIN32
# include <process.h>
#else
# include <unistd.h>
#endif
namespace hostjit
{
JITCompiler::JITCompiler()
: config_(detectDefaultConfig())
{}
JITCompiler::JITCompiler(const CompilerConfig& config)
: config_(config)
{}
JITCompiler::~JITCompiler()
{
cleanup();
}
bool JITCompiler::compile(const std::string& source_code)
{
std::string config_error;
if (!validateConfig(config_, &config_error))
{
last_error_ = "Configuration error: " + config_error;
return false;
}
cleanup();
temp_dir_ = createTempDirectory();
if (temp_dir_.empty())
{
last_error_ = "Failed to create temporary directory";
return false;
}
std::string obj_path = temp_dir_ + "/cuda_code.o";
auto compile_result = compiler_.compileToObject(source_code, obj_path, config_);
if (!compile_result.success)
{
last_error_ = "Compilation failed:\n" + compile_result.diagnostics;
removeTempDirectory();
return false;
}
// Store the cubin for later inspection
cubin_ = std::move(compile_result.cubin);
if (config_.verbose)
{
std::cout << "Compilation diagnostics:\n" << compile_result.diagnostics << "\n";
}
#ifdef _WIN32
std::string lib_path = temp_dir_ + "/cuda_code.dll";
#else
std::string lib_path = temp_dir_ + "/libcuda_code.so";
#endif
auto link_result = compiler_.linkToSharedLibrary({obj_path}, lib_path, config_);
if (!link_result.success)
{
last_error_ = "Linking failed:\n" + link_result.diagnostics;
removeTempDirectory();
return false;
}
if (config_.verbose)
{
std::cout << "Linking diagnostics:\n" << link_result.diagnostics << "\n";
}
if (!library_.load(lib_path))
{
last_error_ = "Failed to load library: " + library_.getLastError();
removeTempDirectory();
return false;
}
if (config_.verbose)
{
std::cout << "Successfully loaded library: " << lib_path << "\n";
}
last_error_.clear();
return true;
}
void JITCompiler::cleanup()
{
library_.unload();
if (!config_.keep_artifacts)
{
removeTempDirectory();
}
last_error_.clear();
}
std::string JITCompiler::createTempDirectory()
{
std::filesystem::path base_tmp_dir;
#ifdef _WIN32
const char* tmp_dir = std::getenv("TEMP");
if (!tmp_dir)
{
tmp_dir = std::getenv("TMP");
}
if (tmp_dir)
{
base_tmp_dir = tmp_dir;
}
else
{
base_tmp_dir = std::filesystem::temp_directory_path();
}
#else
const char* tmp_dir = std::getenv("TMPDIR");
if (tmp_dir)
{
base_tmp_dir = tmp_dir;
}
else
{
base_tmp_dir = "/tmp";
}
#endif
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<> dis(0, 999999);
#ifdef _WIN32
int pid = _getpid();
#else
int pid = getpid();
#endif
for (int attempt = 0; attempt < 10; ++attempt)
{
std::string dir_name = "hostjit_" + std::to_string(pid) + "_" + std::to_string(dis(gen));
std::filesystem::path full_path = base_tmp_dir / dir_name;
std::error_code ec;
if (std::filesystem::create_directories(full_path, ec) && !ec)
{
return full_path.string();
}
}
return "";
}
void JITCompiler::removeTempDirectory()
{
if (temp_dir_.empty())
{
return;
}
try
{
if (std::filesystem::exists(temp_dir_))
{
std::filesystem::remove_all(temp_dir_);
}
}
catch (const std::filesystem::filesystem_error& e)
{
if (config_.verbose)
{
std::cerr << "Warning: Failed to remove temporary directory: " << e.what() << "\n";
}
}
temp_dir_.clear();
}
} // namespace hostjit

210
cccl_upstream/c/parallel.v2/src/hostjit/loader.cpp Normal file
View File

@@ -0,0 +1,210 @@
#include <hostjit/loader.hpp>
#ifdef _WIN32
# define WIN32_LEAN_AND_MEAN
# include <windows.h>
#else
# include <dlfcn.h>
#endif
namespace hostjit
{
#ifdef _WIN32
namespace
{
// Run C++ static constructors in a DLL loaded with /NOENTRY /NODEFAULTLIB.
// The compiler places CUDA fatbin registration in the .CRT$XCU section.
// Without CRT startup, these never run, so we walk the merged .CRT section
// in the PE and call each non-null function pointer.
void runStaticInitializers(HMODULE module)
{
auto base = reinterpret_cast<const unsigned char*>(module);
auto dos = reinterpret_cast<const IMAGE_DOS_HEADER*>(base);
auto nt = reinterpret_cast<const IMAGE_NT_HEADERS*>(base + dos->e_lfanew);
auto sec = IMAGE_FIRST_SECTION(nt);
for (WORD i = 0; i < nt->FileHeader.NumberOfSections; ++i, ++sec)
{
if (memcmp(sec->Name, ".CRT", 4) == 0)
{
using InitFunc = void(__cdecl*)();
auto funcs = reinterpret_cast<InitFunc*>(const_cast<unsigned char*>(base) + sec->VirtualAddress);
size_t count = sec->SizeOfRawData / sizeof(InitFunc);
for (size_t j = 0; j < count; ++j)
{
if (funcs[j])
{
funcs[j]();
}
}
}
}
}
std::string getWindowsError()
{
DWORD error = GetLastError();
if (error == 0)
{
return "";
}
LPSTR buffer = nullptr;
DWORD size = FormatMessageA(
FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
nullptr,
error,
MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
reinterpret_cast<LPSTR>(&buffer),
0,
nullptr);
std::string message;
if (size > 0 && buffer)
{
message = std::string(buffer, size);
while (!message.empty() && (message.back() == '\n' || message.back() == '\r'))
{
message.pop_back();
}
LocalFree(buffer);
}
else
{
message = "Unknown error (code: " + std::to_string(error) + ")";
}
return message;
}
} // anonymous namespace
#endif
DynamicLibrary::DynamicLibrary()
: handle_(nullptr)
{}
DynamicLibrary::~DynamicLibrary()
{
unload();
}
DynamicLibrary::DynamicLibrary(DynamicLibrary&& other) noexcept
: handle_(other.handle_)
, last_error_(std::move(other.last_error_))
{
other.handle_ = nullptr;
}
DynamicLibrary& DynamicLibrary::operator=(DynamicLibrary&& other) noexcept
{
if (this != &other)
{
unload();
handle_ = other.handle_;
last_error_ = std::move(other.last_error_);
other.handle_ = nullptr;
}
return *this;
}
bool DynamicLibrary::load(const std::string& library_path)
{
unload();
#ifdef _WIN32
SetLastError(0);
handle_ = static_cast<void*>(LoadLibraryA(library_path.c_str()));
if (!handle_)
{
last_error_ = getWindowsError();
if (last_error_.empty())
{
last_error_ = "Unknown LoadLibrary error";
}
return false;
}
// The DLL is linked with /NOENTRY (no CRT startup), so C++ static
// constructors (e.g. CUDA fatbin registration) haven't run yet.
runStaticInitializers(static_cast<HMODULE>(handle_));
#else
dlerror();
handle_ = dlopen(library_path.c_str(), RTLD_LAZY | RTLD_LOCAL);
if (!handle_)
{
const char* error = dlerror();
last_error_ = error ? error : "Unknown dlopen error";
return false;
}
#endif
last_error_.clear();
return true;
}
void* DynamicLibrary::getSymbol(const std::string& symbol_name)
{
if (!handle_)
{
last_error_ = "Library not loaded";
return nullptr;
}
#ifdef _WIN32
SetLastError(0);
void* symbol = reinterpret_cast<void*>(GetProcAddress(static_cast<HMODULE>(handle_), symbol_name.c_str()));
if (!symbol)
{
last_error_ = getWindowsError();
if (last_error_.empty())
{
last_error_ = "Symbol not found: " + symbol_name;
}
return nullptr;
}
#else
dlerror();
void* symbol = dlsym(handle_, symbol_name.c_str());
const char* error = dlerror();
if (error)
{
last_error_ = error;
return nullptr;
}
#endif
last_error_.clear();
return symbol;
}
bool DynamicLibrary::isLoaded() const
{
return handle_ != nullptr;
}
std::string DynamicLibrary::getLastError() const
{
return last_error_;
}
void DynamicLibrary::unload()
{
if (handle_)
{
// Intentionally do NOT unload (dlclose / FreeLibrary) a compiled JIT module. See #9367.
//
// Each JIT .so is built by Clang with the classic fatbin embedding (-fcuda-include-gpubinary),
// which emits a module ctor (__cuda_module_ctor -> __cudaRegisterFatBinary)
// in .init_array but NO matching unregister dtor (.fini_array / __cudaUnregisterFatBinary).
// Unloading such a module unmaps its fatbin while the CUDA runtime still holds a pointer to it;
// that dangling registration corrupts the runtime's module table, so a later module's kernel
// launch silently no-ops.
handle_ = nullptr;
}
last_error_.clear();
}
} // namespace hostjit

222
cccl_upstream/c/parallel.v2/src/merge_sort.cu Normal file
View File

@@ -0,0 +1,222 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#include <cstdio>
#include <cstring>
#include <cccl/c/merge_sort.h>
#include <hostjit/codegen/cub_call.hpp>
#include <util/build_utils.h>
using namespace hostjit::codegen;
// Keys-only: (temp, temp_bytes, in_keys, out_keys, num_items, cmp_state, stream)
using keys_fn_t = int (*)(void*, size_t*, void*, void*, unsigned long long, void*, void*);
// Key-value pairs: (temp, temp_bytes, in_keys, in_items, out_keys, out_items, num_items, cmp_state, stream)
using pairs_fn_t = int (*)(void*, size_t*, void*, void*, void*, void*, unsigned long long, void*, void*);
static bool is_null_items(cccl_iterator_t it)
{
return it.type == CCCL_POINTER && it.state == nullptr;
}
// ---------------------------------------------------------------------------
// Build
// ---------------------------------------------------------------------------
CUresult cccl_device_merge_sort_build_ex(
cccl_device_merge_sort_build_result_t* build_ptr,
cccl_iterator_t d_in_keys,
cccl_iterator_t d_in_items,
cccl_iterator_t d_out_keys,
cccl_iterator_t d_out_items,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config)
try
{
if (d_out_keys.type == CCCL_ITERATOR || d_out_items.type == CCCL_ITERATOR)
{
fprintf(stderr, "\nERROR in cccl_device_merge_sort_build(): merge sort output cannot be an iterator\n");
return CUDA_ERROR_UNKNOWN;
}
std::string cccl_include_str = cccl::detail::parse_cccl_include_path(libcudacxx_path);
std::string ctk_root_str = cccl::detail::parse_ctk_root(ctk_path);
const char* cccl_include_path = cccl_include_str.empty() ? nullptr : cccl_include_str.c_str();
const char* ctk_root = ctk_root_str.empty() ? nullptr : ctk_root_str.c_str();
cccl::detail::MergedBuildConfig merged(config, cub_path, thrust_path);
const bool has_items = !is_null_items(d_in_items);
CubCallResult result = [&] {
if (has_items)
{
return CubCall::from("cub/device/device_merge_sort.cuh")
.run("cub::DeviceMergeSort::SortPairsCopy")
.name("cccl_jit_merge_sort")
.with(temp_storage,
temp_bytes,
in(d_in_keys),
in(d_in_items),
out(d_out_keys),
out(d_out_items),
num_items,
cmp(op),
stream)
.compile(cc_major, cc_minor, merged.get(), ctk_root, cccl_include_path);
}
else
{
return CubCall::from("cub/device/device_merge_sort.cuh")
.run("cub::DeviceMergeSort::SortKeysCopy")
.name("cccl_jit_merge_sort")
.with(temp_storage, temp_bytes, in(d_in_keys), out(d_out_keys), num_items, cmp(op), stream)
.compile(cc_major, cc_minor, merged.get(), ctk_root, cccl_include_path);
}
}();
build_ptr->cc = cc_major * 10 + cc_minor;
cccl::detail::copy_cubin(result.cubin, build_ptr->payload, build_ptr->payload_size);
build_ptr->jit_compiler = result.compiler;
build_ptr->sort_fn = result.fn_ptr;
build_ptr->keys_only = has_items ? 0 : 1;
build_ptr->key_type = d_in_keys.value_type;
build_ptr->item_type = d_in_items.value_type;
return CUDA_SUCCESS;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_merge_sort_build(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}
CUresult cccl_device_merge_sort_build(
cccl_device_merge_sort_build_result_t* build,
cccl_iterator_t d_in_keys,
cccl_iterator_t d_in_items,
cccl_iterator_t d_out_keys,
cccl_iterator_t d_out_items,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path)
{
return cccl_device_merge_sort_build_ex(
build,
d_in_keys,
d_in_items,
d_out_keys,
d_out_items,
op,
cc_major,
cc_minor,
cub_path,
thrust_path,
libcudacxx_path,
ctk_path,
nullptr);
}
// ---------------------------------------------------------------------------
// Run
// ---------------------------------------------------------------------------
CUresult cccl_device_merge_sort(
cccl_device_merge_sort_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in_keys,
cccl_iterator_t d_in_items,
cccl_iterator_t d_out_keys,
cccl_iterator_t d_out_items,
uint64_t num_items,
cccl_op_t op,
CUstream stream)
try
{
if (!build.sort_fn)
{
return CUDA_ERROR_INVALID_VALUE;
}
int status;
// Dispatch to the correct function arity. build.keys_only was set at
// build time, so we don't have to re-derive the pairs-vs-keys decision
// from the iterator arguments here (and they must match the build for
// the function-pointer types to be valid).
if (!build.keys_only)
{
// Pairs build: (temp, temp_bytes, in_keys, in_items, out_keys, out_items, num_items, cmp_state, stream)
auto fn = reinterpret_cast<pairs_fn_t>(build.sort_fn);
status = fn(
d_temp_storage,
temp_storage_bytes,
d_in_keys.state,
d_in_items.state,
d_out_keys.state,
d_out_items.state,
num_items,
op.state,
reinterpret_cast<void*>(stream));
}
else
{
// Keys-only build: (temp, temp_bytes, in_keys, out_keys, num_items, cmp_state, stream)
auto fn = reinterpret_cast<keys_fn_t>(build.sort_fn);
status =
fn(d_temp_storage,
temp_storage_bytes,
d_in_keys.state,
d_out_keys.state,
num_items,
op.state,
reinterpret_cast<void*>(stream));
}
return (status == 0) ? CUDA_SUCCESS : CUDA_ERROR_UNKNOWN;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_merge_sort(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}
// ---------------------------------------------------------------------------
// Cleanup
// ---------------------------------------------------------------------------
CUresult cccl_device_merge_sort_cleanup(cccl_device_merge_sort_build_result_t* build_ptr)
try
{
if (build_ptr == nullptr)
{
return CUDA_ERROR_INVALID_VALUE;
}
cccl::detail::release_jit_artifacts(build_ptr);
build_ptr->sort_fn = nullptr;
return CUDA_SUCCESS;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_merge_sort_cleanup(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}

356
cccl_upstream/c/parallel.v2/src/radix_sort.cu Normal file
View File

@@ -0,0 +1,356 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#include <cstdio>
#include <cccl/c/radix_sort.h>
#include <hostjit/codegen/cub_call.hpp>
#include <util/build_utils.h>
using namespace hostjit::codegen;
static bool is_null_it(cccl_iterator_t it)
{
return it.type == CCCL_POINTER && it.state == nullptr;
}
static bool is_null_op(cccl_op_t op)
{
return op.name == nullptr || op.name[0] == '\0';
}
// Two JIT wrappers are produced by CubCall per build:
//
// COPY variant — wraps cub::DeviceRadixSort::Sort{Keys,Pairs}{,Descending}'s
// copy-overload. Result is always in *_out; selector is implicitly 0. Used
// when the caller invokes the run-time API with is_overwrite_okay=false.
// keys-only: fn(temp, temp_bytes, keys_in, keys_out, num_items,
// &begin_bit, &end_bit, stream)
// pairs: fn(temp, temp_bytes, keys_in, keys_out, values_in, values_out,
// num_items, &begin_bit, &end_bit, stream)
//
// DOUBLE-BUFFER (overwrite) variant — wraps the DoubleBuffer overload.
// Constructs cub::DoubleBuffer<KeyT>(keys_in, keys_out) (and ValueT for
// pairs), runs the sort, then writes the buffer's `selector` (0 or 1) to a
// host-provided int*. Result may live in either keys_in or keys_out depending
// on the number of CUB passes — the selector tells the caller which.
// keys-only: fn(temp, temp_bytes, keys_in, keys_out, num_items,
// &begin_bit, &end_bit, selector_out, stream)
// pairs: fn(temp, temp_bytes, keys_in, keys_out, values_in, values_out,
// num_items, &begin_bit, &end_bit, selector_out, stream)
//
// begin_bit/end_bit go through CubCall::typed_scalar (host-pointer + memcpy
// onto the stack inside the JIT wrapper).
//
// Decomposer: only identity (null decomposer) is supported.
using radix_sort_keys_fn_t = int (*)(void*, size_t*, void*, void*, unsigned long long, void*, void*, void*);
using radix_sort_pairs_fn_t =
int (*)(void*, size_t*, void*, void*, void*, void*, unsigned long long, void*, void*, void*);
using radix_sort_keys_overwrite_fn_t =
int (*)(void*, size_t*, void*, void*, unsigned long long, void*, void*, void*, void*);
using radix_sort_pairs_overwrite_fn_t =
int (*)(void*, size_t*, void*, void*, void*, void*, unsigned long long, void*, void*, void*, void*);
// Type info for the begin_bit/end_bit int scalars passed to CubCall.
static constexpr cccl_type_info k_int_type{sizeof(int), alignof(int), CCCL_INT32};
CUresult cccl_device_radix_sort_build_ex(
cccl_device_radix_sort_build_result_t* build_ptr,
cccl_sort_order_t sort_order,
cccl_iterator_t input_keys_it,
cccl_iterator_t input_values_it,
cccl_op_t decomposer,
const char* /*decomposer_return_type*/,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config)
try
{
if (!is_null_op(decomposer))
{
fprintf(stderr,
"\nERROR in cccl_device_radix_sort_build(): custom radix decomposers are not supported "
"in the HostJIT path. Use standard integer/float key types.\n");
return CUDA_ERROR_UNKNOWN;
}
std::string cccl_include_str = cccl::detail::parse_cccl_include_path(libcudacxx_path);
std::string ctk_root_str = cccl::detail::parse_ctk_root(ctk_path);
const char* cccl_include_path = cccl_include_str.empty() ? nullptr : cccl_include_str.c_str();
const char* ctk_root = ctk_root_str.empty() ? nullptr : ctk_root_str.c_str();
cccl::detail::MergedBuildConfig merged(config, cub_path, thrust_path);
const bool keys_only = is_null_it(input_values_it);
const bool ascending = (sort_order == CCCL_ASCENDING);
// CUB writes to caller-provided device pointers at run time. Build needs
// an iterator descriptor for the outputs; synthesize raw-pointer ones with
// the same value_type as the matching input.
cccl_iterator_t output_keys_it = input_keys_it;
output_keys_it.type = CCCL_POINTER;
output_keys_it.state = nullptr;
cccl_iterator_t output_values_it{};
output_values_it.type = CCCL_POINTER;
output_values_it.state = nullptr;
output_values_it.value_type = input_values_it.value_type;
const char* cub_algo;
if (keys_only)
{
cub_algo = ascending ? "cub::DeviceRadixSort::SortKeys" : "cub::DeviceRadixSort::SortKeysDescending";
}
else
{
cub_algo = ascending ? "cub::DeviceRadixSort::SortPairs" : "cub::DeviceRadixSort::SortPairsDescending";
}
auto cb_copy = [&] {
if (keys_only)
{
return CubCall::from("cub/device/device_radix_sort.cuh")
.run(cub_algo)
.name("cccl_jit_radix_sort")
.with(temp_storage,
temp_bytes,
in(input_keys_it),
out(output_keys_it),
num_items,
typed_scalar(k_int_type, "begin_bit"),
typed_scalar(k_int_type, "end_bit"),
stream);
}
return CubCall::from("cub/device/device_radix_sort.cuh")
.run(cub_algo)
.name("cccl_jit_radix_sort")
.with(temp_storage,
temp_bytes,
in(input_keys_it),
out(output_keys_it),
in(input_values_it),
out(output_values_it),
num_items,
typed_scalar(k_int_type, "begin_bit"),
typed_scalar(k_int_type, "end_bit"),
stream);
}();
auto cb_overwrite = [&] {
if (keys_only)
{
return CubCall::from("cub/device/device_radix_sort.cuh")
.run(cub_algo)
.name("cccl_jit_radix_sort_overwrite")
.with(temp_storage,
temp_bytes,
double_buffer(input_keys_it, output_keys_it, "d_keys_buffer"),
num_items,
typed_scalar(k_int_type, "begin_bit"),
typed_scalar(k_int_type, "end_bit"),
selector_out("d_keys_buffer"),
stream);
}
return CubCall::from("cub/device/device_radix_sort.cuh")
.run(cub_algo)
.name("cccl_jit_radix_sort_overwrite")
.with(temp_storage,
temp_bytes,
double_buffer(input_keys_it, output_keys_it, "d_keys_buffer"),
double_buffer(input_values_it, output_values_it, "d_values_buffer"),
num_items,
typed_scalar(k_int_type, "begin_bit"),
typed_scalar(k_int_type, "end_bit"),
selector_out("d_keys_buffer"),
stream);
}();
auto result =
CubCall::compile({cb_copy, cb_overwrite}, cc_major, cc_minor, merged.get(), ctk_root, cccl_include_path);
build_ptr->cc = cc_major * 10 + cc_minor;
cccl::detail::copy_cubin(result.cubin, build_ptr->payload, build_ptr->payload_size);
build_ptr->jit_compiler = result.compiler;
build_ptr->sort_fn = result.fn_ptrs[0];
build_ptr->sort_fn_overwrite = result.fn_ptrs[1];
build_ptr->key_type = input_keys_it.value_type;
build_ptr->value_type = input_values_it.value_type;
build_ptr->order = sort_order;
build_ptr->keys_only = keys_only ? 1 : 0;
return CUDA_SUCCESS;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_radix_sort_build(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}
CUresult cccl_device_radix_sort_build(
cccl_device_radix_sort_build_result_t* build,
cccl_sort_order_t sort_order,
cccl_iterator_t input_keys_it,
cccl_iterator_t input_values_it,
cccl_op_t decomposer,
const char* decomposer_return_type,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path)
{
return cccl_device_radix_sort_build_ex(
build,
sort_order,
input_keys_it,
input_values_it,
decomposer,
decomposer_return_type,
cc_major,
cc_minor,
cub_path,
thrust_path,
libcudacxx_path,
ctk_path,
nullptr);
}
CUresult cccl_device_radix_sort(
cccl_device_radix_sort_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_keys_out,
cccl_iterator_t d_values_in,
cccl_iterator_t d_values_out,
cccl_op_t /*decomposer*/,
uint64_t num_items,
int begin_bit,
int end_bit,
bool is_overwrite_okay,
int* selector,
CUstream stream)
try
{
// Dispatch on is_overwrite_okay: the copy variant always lands the result
// in d_keys_out (selector = 0); the DoubleBuffer variant may land it in
// either buffer and reports which via its `selector` member, captured here
// by passing a pointer for the wrapper to write into.
int status;
int local_selector = 0;
if (is_overwrite_okay)
{
if (!build.sort_fn_overwrite)
{
return CUDA_ERROR_INVALID_VALUE;
}
if (build.keys_only)
{
auto fn = reinterpret_cast<radix_sort_keys_overwrite_fn_t>(build.sort_fn_overwrite);
status = fn(
d_temp_storage,
temp_storage_bytes,
d_keys_in.state,
d_keys_out.state,
static_cast<unsigned long long>(num_items),
&begin_bit,
&end_bit,
&local_selector,
reinterpret_cast<void*>(stream));
}
else
{
auto fn = reinterpret_cast<radix_sort_pairs_overwrite_fn_t>(build.sort_fn_overwrite);
status = fn(
d_temp_storage,
temp_storage_bytes,
d_keys_in.state,
d_keys_out.state,
d_values_in.state,
d_values_out.state,
static_cast<unsigned long long>(num_items),
&begin_bit,
&end_bit,
&local_selector,
reinterpret_cast<void*>(stream));
}
}
else
{
if (!build.sort_fn)
{
return CUDA_ERROR_INVALID_VALUE;
}
if (build.keys_only)
{
auto fn = reinterpret_cast<radix_sort_keys_fn_t>(build.sort_fn);
status = fn(
d_temp_storage,
temp_storage_bytes,
d_keys_in.state,
d_keys_out.state,
static_cast<unsigned long long>(num_items),
&begin_bit,
&end_bit,
reinterpret_cast<void*>(stream));
}
else
{
auto fn = reinterpret_cast<radix_sort_pairs_fn_t>(build.sort_fn);
status = fn(
d_temp_storage,
temp_storage_bytes,
d_keys_in.state,
d_keys_out.state,
d_values_in.state,
d_values_out.state,
static_cast<unsigned long long>(num_items),
&begin_bit,
&end_bit,
reinterpret_cast<void*>(stream));
}
local_selector = 0;
}
if (selector)
{
*selector = local_selector;
}
return (status == 0) ? CUDA_SUCCESS : CUDA_ERROR_UNKNOWN;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_radix_sort(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}
CUresult cccl_device_radix_sort_cleanup(cccl_device_radix_sort_build_result_t* build_ptr)
try
{
if (build_ptr == nullptr)
{
return CUDA_ERROR_INVALID_VALUE;
}
cccl::detail::release_jit_artifacts(build_ptr);
build_ptr->sort_fn = nullptr;
build_ptr->sort_fn_overwrite = nullptr;
return CUDA_SUCCESS;
}
catch (const std::exception& exc)
{
fprintf(stderr, "\nEXCEPTION in cccl_device_radix_sort_cleanup(): %s\n", exc.what());
return CUDA_ERROR_UNKNOWN;
}

Some files were not shown because too many files have changed in this diff Show More