diff --git a/cccl_upstream/cudax/CMakeLists.txt b/cccl_upstream/cudax/CMakeLists.txt deleted file mode 100644 index 728ffb55..00000000 --- a/cccl_upstream/cudax/CMakeLists.txt +++ /dev/null @@ -1,94 +0,0 @@ -if (NOT CCCL_ENABLE_CUDAX) - include(cmake/cudaxAddSubdir.cmake) - return() -endif() - -cmake_minimum_required(VERSION 3.21) -project(cudax LANGUAGES CXX CUDA) - -option( - cudax_ENABLE_HEADER_TESTING - "Test that CUDA Experimental's public headers compile." - ON -) -option(cudax_ENABLE_TESTING "Build CUDA Experimental's tests." ON) -option(cudax_ENABLE_EXAMPLES "Build CUDA Experimental's examples." ON) -option(cudax_ENABLE_PLACES "Enable standalone Places subproject" ON) -option(cudax_ENABLE_CUDASTF "Enable CUDASTF subproject" ON) -option( - cudax_ENABLE_CUDASTF_CODE_GENERATION - "Enable code generation using STF's parallel_for or launch with CUDA compiler." - ON -) -option( - cudax_ENABLE_CUDASTF_BOUNDSCHECK - "Enable bounds checks for STF targets. Requires debug build." - OFF -) -option( - cudax_ENABLE_CUDASTF_MATHLIBS - "Enable STF tests/examples that use cublas/cusolver." - OFF -) -option(cudax_ENABLE_CUFILE "Enable cuFile in CUDA Experimental" ON) - -if (cudax_ENABLE_CUFILE) - if (WIN32) - message(FATAL_ERROR "cuFile is not available on Windows.") - endif() - - if (CMAKE_VERSION VERSION_LESS "3.25.0") - message( - FATAL_ERROR - "cuFile is not available before cmake 3.25.0, please, use newer cmake." - ) - endif() - - cccl_get_cudatoolkit() - - if (CUDAToolkit_VERSION VERSION_LESS "12.9.0") - message(FATAL_ERROR "cuFile support requires at least CUDA 12.9.") - endif() - - if (NOT TARGET CUDA::cuFile) - message( - FATAL_ERROR - "CUDA::cuFile target required for requested cuFile support was not found." - ) - endif() -endif() - -if ( - cudax_ENABLE_CUDASTF_BOUNDSCHECK - AND NOT CMAKE_BUILD_TYPE MATCHES "Debug" - AND NOT CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo" -) - message( - FATAL_ERROR - "cudax_ENABLE_CUDASTF_BOUNDSCHECK requires a Debug build." - ) -endif() - -include(cmake/cudaxBuildCompilerTargets.cmake) -if (cudax_ENABLE_PLACES) - include(cmake/cudaxPlacesConfigureTarget.cmake) -endif() -if (cudax_ENABLE_CUDASTF) - include(cmake/cudaxSTFConfigureTarget.cmake) -endif() - -if (cudax_ENABLE_HEADER_TESTING) - include(cmake/cudaxHeaderTesting.cmake) -endif() - -if (cudax_ENABLE_TESTING) - add_subdirectory(test) -endif() - -if (cudax_ENABLE_EXAMPLES) - add_subdirectory(examples) -endif() - -if (CCCL_ENABLE_BENCHMARKS) - add_subdirectory(benchmarks) -endif() diff --git a/cccl_upstream/cudax/LICENSE.TXT b/cccl_upstream/cudax/LICENSE.TXT deleted file mode 100644 index fba5f88f..00000000 --- a/cccl_upstream/cudax/LICENSE.TXT +++ /dev/null @@ -1,222 +0,0 @@ -==================================================================================== -The CUDA Experimental library is under the Apache License v2.0 with LLVM Exceptions: -==================================================================================== - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - ----- LLVM Exceptions to the Apache 2.0 License ---- - -As an exception, if, as a result of your compiling your source code, portions -of this Software are embedded into an Object form of such source code, you -may redistribute such embedded portions in such Object form without complying -with the conditions of Sections 4(a), 4(b) and 4(d) of the License. - -In addition, if you combine or link compiled forms of this Software with -software that is licensed under the GPLv2 ("Combined Software") and if a -court of competent jurisdiction determines that the patent provision (Section -3), the indemnity provision (Section 9) or other Section of the License -conflicts with the conditions of the GPLv2, you may retroactively and -prospectively choose to deem waived or otherwise exclude such Section(s) of -the License, but only in their entirety and only with respect to the Combined -Software. diff --git a/cccl_upstream/cudax/README.md b/cccl_upstream/cudax/README.md deleted file mode 100644 index 522d5ea7..00000000 --- a/cccl_upstream/cudax/README.md +++ /dev/null @@ -1,32 +0,0 @@ -## CUDA Experimental: Library for experimental features in CUDA Core Compute Libraries. -CUDA Experimental serves as a distribution channel for features that are considered experimental in the CUDA Core Compute Libraries. -Some of them are still actively designed or developed and their API is evolving. -Some of them are specific to one hardware architecture and are still looking for a generic and forward compatible exposure. -Finally, some of them need to prove useful enough to deserve long term support. - -**All APIs available in CUDA Experimental are not considered stable and can change without a notice.** They can also be deprecated or removed on a much faster cadence than in other CCCL libraries. - -Features are exposed here for the CUDA C++ community to experiment with and provide feedback on how to shape it to best fit their use cases. -Once we become confident a feature is ready and would be a great permanent addition in CCCL, it will become a part of some other CCCL library with a stable API. - -## Installation -CUDA Experimental library is **not** distributed with the CUDA Toolkit like the rest of CCCL. It is only available on the [CCCL GitHub repository](https://github.com/NVIDIA/cccl). - -CUDA Experimental compilation requires C++17 standard or newer. Supported compilers are: - -CUDA Compilers: -- NVCC 12.3+ - -NVCC host compilers: -- GCC 7+ -- Clang 9+ -- MSVC 2019+ - -Everything in CUDA Experimental is header-only, so cloning and including it in a simple project is as easy as the following: -```bash -git clone https://github.com/NVIDIA/cccl.git -# Note: -nvcc -Icccl/cudax/include main.cu -o main -``` - -A CMake target `cudax::cudax` is available as part of the CCCL package when `CCCL_ENABLE_UNSTABLE` is set to a truthy value before calling `find_package` or `add_subdirectory`. diff --git a/cccl_upstream/cudax/benchmarks/CMakeLists.txt b/cccl_upstream/cudax/benchmarks/CMakeLists.txt deleted file mode 100644 index 5134efc6..00000000 --- a/cccl_upstream/cudax/benchmarks/CMakeLists.txt +++ /dev/null @@ -1,69 +0,0 @@ -include(${CMAKE_SOURCE_DIR}/benchmarks/cmake/CCCLBenchmarkRegistry.cmake) - -cccl_get_nvbench() -cccl_get_nvbench_helper() - -set(benches_root "${CMAKE_CURRENT_LIST_DIR}") - -function(get_recursive_subdirs subdirs) - set(dirs) - file( - GLOB_RECURSE contents - CONFIGURE_DEPENDS - LIST_DIRECTORIES ON - "${CMAKE_CURRENT_LIST_DIR}/bench/*" - ) - - foreach (bench_dir IN LISTS contents) - if (IS_DIRECTORY "${bench_dir}") - list(APPEND dirs "${bench_dir}") - endif() - endforeach() - - set(${subdirs} "${dirs}" PARENT_SCOPE) -endfunction() - -function(add_bench target_name bench_name bench_src) - set(bench_target ${bench_name}) - set(${target_name} ${bench_target} PARENT_SCOPE) - - cccl_add_executable(${bench_target} SOURCES "${bench_src}") - target_link_libraries( - ${bench_target} - PRIVATE # - cccl.nvbench_helper - nvbench::main - ) -endfunction() - -function(add_bench_dir bench_dir) - file(GLOB bench_srcs CONFIGURE_DEPENDS "${bench_dir}/*.cu") - file(RELATIVE_PATH bench_prefix "${benches_root}" "${bench_dir}") - file(TO_CMAKE_PATH "${bench_prefix}" bench_prefix) - string(REPLACE "/" "." bench_prefix "${bench_prefix}") - - foreach (bench_src IN LISTS bench_srcs) - get_filename_component(bench_name "${bench_src}" NAME_WLE) - string(PREPEND bench_name "cudax.${bench_prefix}.") - register_cccl_benchmark("${bench_name}" "") - - string(APPEND bench_name ".base") - - add_bench(base_bench_target ${bench_name} "${bench_src}") - target_link_libraries(${bench_name} PRIVATE cudax.compiler_interface) - target_compile_options( - ${bench_name} - PRIVATE - "$<$:--extended-lambda>" - # cudax.compiler_interface enables assertions for tests/examples; benchmarks should measure release behavior. - "$<$:-UCCCL_ENABLE_ASSERTIONS>" - "$<$:-UCCCL_ENABLE_ASSERTIONS>" - ) - endforeach() -endfunction() - -get_recursive_subdirs(subdirs) - -foreach (subdir IN LISTS subdirs) - add_bench_dir("${subdir}") -endforeach() diff --git a/cccl_upstream/cudax/benchmarks/bench/copy/copy_bench.cu b/cccl_upstream/cudax/benchmarks/bench/copy/copy_bench.cu deleted file mode 100644 index c049d57a..00000000 --- a/cccl_upstream/cudax/benchmarks/bench/copy/copy_bench.cu +++ /dev/null @@ -1,331 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -#include - -#include -#include -#include - -#include - -#include -#include -#include - -#include - -// GCC -Warray-bounds false positive for high-rank (20+) __raw_tensor instantiations -_CCCL_DIAG_SUPPRESS_GCC("-Warray-bounds") - -template -size_t -compute_alloc(size_t offset, const cuda::std::array& shape, const cuda::std::array& strides) -{ - int64_t max_pos = static_cast(offset); - for (size_t i = 0; i < Rank; ++i) - { - auto delta = static_cast(shape[i] - 1) * strides[i]; - if (delta > 0) - { - max_pos += delta; - } - } - return max_pos + 1; -} - -template -void bench_copy(nvbench::state& state, - size_t src_offset, - const cuda::std::array& shape, - const cuda::std::array& src_strides, - size_t dst_offset, - const cuda::std::array& dst_strides) -{ - const auto src_alloc = compute_alloc(src_offset, shape, src_strides); - const auto dst_alloc = compute_alloc(dst_offset, shape, dst_strides); - - thrust::device_vector d_src(src_alloc); - thrust::device_vector d_dst(dst_alloc); - - size_t num_items = 1; - for (size_t i = 0; i < Rank; ++i) - { - num_items *= shape[i]; - } - state.add_element_count(num_items); - state.add_global_memory_reads(num_items); - state.add_global_memory_writes(num_items); - - using extents_t = cuda::std::dextents; - using strides_t = cuda::dstrides; - using mapping_t = cuda::layout_stride_relaxed::mapping; - - extents_t ext(shape); - auto src_ptr = thrust::raw_pointer_cast(d_src.data()) + src_offset; - auto dst_ptr = thrust::raw_pointer_cast(d_dst.data()) + dst_offset; - mapping_t src_map(ext, strides_t(src_strides)); - mapping_t dst_map(ext, strides_t(dst_strides)); - - cuda::device_mdspan src(src_ptr, src_map); - cuda::device_mdspan dst(dst_ptr, dst_map); - - state.exec([&](nvbench::launch& launch) { - cuda::stream_ref stream{launch.get_stream()}; - cuda::experimental::copy(src, dst, stream); - }); -} - -template -void bench_copy(nvbench::state& state, - size_t offset, - const cuda::std::array& shape, - const cuda::std::array& strides) -{ - bench_copy(state, offset, shape, strides, offset, strides); -} - -/*********************************************************************************************************************** - * Memcpy benchmarks - **********************************************************************************************************************/ - -// src: (25, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1) -// dst: (25, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1) -void memcpy_layout_0(nvbench::state& state) -{ - cuda::std::array shape{25, 70, 90, 80, 80}; - cuda::std::array strides{40320000, 576000, 6400, 80, 1}; - bench_copy(state, 0, shape, strides); -} -NVBENCH_BENCH(memcpy_layout_0).set_name("contiguous (5D, int, 4GB)"); - -// src: (25, 80, 70, 80, 90):(40320000, 1, 576000, 80, 6400) -// dst: (25, 80, 70, 80, 90):(40320000, 1, 576000, 80, 6400) -void memcpy_layout_1(nvbench::state& state) -{ - cuda::std::array shape{25, 80, 70, 80, 90}; - cuda::std::array strides{40320000, 1, 576000, 80, 6400}; - bench_copy(state, 0, shape, strides); -} -NVBENCH_BENCH(memcpy_layout_1).set_name("contiguous-perm (5D, int, 4GB)"); - -// src: (1, 25, 1, 80, 1, 70, 1, 80, 1, 90):(1, 40320000, 1, 1, 1, 576000, 1, 80, 1, 6400) -// dst: (1, 25, 1, 80, 1, 70, 1, 80, 1, 90):(1, 40320000, 1, 1, 1, 576000, 1, 80, 1, 6400) -void memcpy_layout_1b(nvbench::state& state) -{ - cuda::std::array shape{1, 25, 1, 80, 1, 70, 1, 80, 1, 90}; - cuda::std::array strides{1, 40320000, 1, 1, 1, 576000, 1, 80, 1, 6400}; - bench_copy(state, 0, shape, strides); -} -NVBENCH_BENCH(memcpy_layout_1b).set_name("contiguous-1-sized (10D, int, 4GB)"); - -// src: (25, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1) -// dst: (25, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1) -void memcpy_layout_2(nvbench::state& state) -{ - cuda::std::array shape{25, 70, 90, 80, 80}; - cuda::std::array strides{40320000, 576000, 6400, 80, 1}; - bench_copy(state, 1, shape, strides); -} -NVBENCH_BENCH(memcpy_layout_2).set_name("contiguous-not-aligned (5D, int, 4GB)"); - -// src: (100, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1) -// dst: (100, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1) -void memcpy_layout_3(nvbench::state& state) -{ - cuda::std::array shape{100, 70, 90, 80, 80}; - cuda::std::array strides{40320000, 576000, 6400, 80, 1}; - bench_copy(state, 0, shape, strides); -} -NVBENCH_BENCH(memcpy_layout_3).set_name("contiguous-small (5D, char, 4GB)"); - -// src: (100, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1) -// dst: (100, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1) -void memcpy_layout_4(nvbench::state& state) -{ - cuda::std::array shape{100, 70, 90, 80, 80}; - cuda::std::array strides{40320000, 576000, 6400, 80, 1}; - bench_copy(state, 1, shape, strides); -} -NVBENCH_BENCH(memcpy_layout_4).set_name("contiguous-small-not-aligned (5D, char, 4GB)"); - -// src: (25, 70, 90, 80, 80):(40320000, 576000, 6400, 80, -1), offset=80 -// dst: (25, 70, 90, 80, 80):(40320000, 576000, 6400, 80, -1), offset=80 -void memcpy_neg(nvbench::state& state) -{ - cuda::std::array shape{25, 70, 90, 80, 80}; - cuda::std::array strides{40320000, 576000, 6400, 80, -1}; - bench_copy(state, 80, shape, strides); -} -NVBENCH_BENCH(memcpy_neg).set_name("contiguous-negative-stride (5D, int, 4GB)"); - -// src: (134217600, 32):(128, 1), offset=32 -// dst: (134217600, 32):(128, 1), offset=32 -// Copies 4GB while allocating 16GB per tensor because of the padded outer stride. -void vectorization(nvbench::state& state) -{ - cuda::std::array shape{134217600, 32}; - cuda::std::array strides{128, 1}; - bench_copy(state, 32, shape, strides); -} -NVBENCH_BENCH(vectorization).set_name("vectorization (2D, char, 4GB copy, 16GB alloc)"); - -// src: (32767, (128 * 1024) / sizeof(int)):(128 * 1024, 1) -// dst: (32767, (128 * 1024) / sizeof(int)):(128 * 1024, 1) -// Copies 4GB while allocating 16GB per tensor because each row is padded to 128K elements. -void block_contiguous(nvbench::state& state) -{ - cuda::std::array shape{32767, (128 * 1024) / sizeof(int)}; - cuda::std::array strides{128 * 1024, 1}; - bench_copy(state, 0, shape, strides); -} -NVBENCH_BENCH(block_contiguous).set_name("block-contiguous (2D, int, 4GB copy, 16GB alloc)"); -// (non-vectorizable) - -void several_dimensions(nvbench::state& state) -{ - cuda::std::array shape{64, 64, 64, 64, 64}; - cuda::std::array strides{17043520 + 1, 266304 + 1, 4160 + 1, 64 + 1, 1}; - bench_copy(state, 0, shape, strides); -} -NVBENCH_BENCH(several_dimensions).set_name("several_dimensions (5D, int, 4GB)"); - -void several_dimensions_non_square(nvbench::state& state) -{ - cuda::std::array shape{63, 65, 67, 69, 57}; - cuda::std::array strides{17433131, 268202, 4003, 58, 1}; - bench_copy(state, 0, shape, strides); -} -NVBENCH_BENCH(several_dimensions_non_square).set_name("several_dimensions_non_square (5D, int, 4GB)"); - -/*********************************************************************************************************************** - * Transpose benchmark - **********************************************************************************************************************/ - -// src: (32768,32768):(1,32768) -// dst: (32768,32768):(32768,1) -void transpose_2D_col_row(nvbench::state& state) -{ - cuda::std::array shape{32768, 32768}; - cuda::std::array src_strides{1, 32768}; - cuda::std::array dst_strides{32768, 1}; - bench_copy(state, 0, shape, src_strides, 0, dst_strides); -} -NVBENCH_BENCH(transpose_2D_col_row).set_name("transpose_2D_col_row (2D, int, 4GB)"); - -void transpose_2D_row_col(nvbench::state& state) -{ - cuda::std::array shape{32768, 32768}; - cuda::std::array src_strides{32768, 1}; - cuda::std::array dst_strides{1, 32768}; - bench_copy(state, 0, shape, src_strides, 0, dst_strides); -} -NVBENCH_BENCH(transpose_2D_row_col).set_name("transpose_2D_row_col (2D, int, 4GB)"); - -void transpose_2D_char(nvbench::state& state) -{ - cuda::std::array shape{65536, 65536}; - cuda::std::array src_strides{1, 65536}; - cuda::std::array dst_strides{65536, 1}; - bench_copy(state, 0, shape, src_strides, 0, dst_strides); -} -NVBENCH_BENCH(transpose_2D_char).set_name("transpose_2D_char (2D, char, 4GB)"); - -void transpose_2D_short(nvbench::state& state) -{ - cuda::std::array shape{32760, 32768 * 2}; - cuda::std::array src_strides{1, 32760}; - cuda::std::array dst_strides{32768 * 2, 1}; - bench_copy(state, 0, shape, src_strides, 0, dst_strides); -} -NVBENCH_BENCH(transpose_2D_short).set_name("transpose_2D_short (2D, short, 4GB)"); - -void transpose_2D_double(nvbench::state& state) -{ - cuda::std::array shape{32768, 16384}; - cuda::std::array src_strides{1, 32768}; - cuda::std::array dst_strides{16384, 1}; - bench_copy(state, 0, shape, src_strides, 0, dst_strides); -} -NVBENCH_BENCH(transpose_2D_double).set_name("transpose_2D_double (2D, double, 4GB)"); - -void transpose_2D_odd_both(nvbench::state& state) -{ - cuda::std::array shape{32767, 32769}; - cuda::std::array src_strides{1, 32767}; - cuda::std::array dst_strides{32769, 1}; - bench_copy(state, 0, shape, src_strides, 0, dst_strides); -} -NVBENCH_BENCH(transpose_2D_odd_both).set_name("transpose_2D_odd_both (2D, int, 4GB)"); - -void transpose_3D(nvbench::state& state) -{ - cuda::std::array shape{1024, 1024, 1024}; - cuda::std::array src_strides{1, 1024, 1024 * 1024}; - cuda::std::array dst_strides{1024 * 1024, 1024, 1}; - bench_copy(state, 0, shape, src_strides, 0, dst_strides); -} -NVBENCH_BENCH(transpose_3D).set_name("transpose_3D (3D, int, 4GB)"); - -void transpose_3D_odd_edges(nvbench::state& state) -{ - cuda::std::array shape{1023, 1025, 1024}; - cuda::std::array src_strides{1, 1023, 1023 * 1025}; - cuda::std::array dst_strides{1025 * 1024, 1024, 1}; - bench_copy(state, 0, shape, src_strides, 0, dst_strides); -} -NVBENCH_BENCH(transpose_3D_odd_edges).set_name("transpose_3D_odd_edges (3D, int, 4GB)"); - -void transpose_src_small_15(nvbench::state& state) -{ - cuda::std::array shape{15, 2236962, 32}; - cuda::std::array src_strides{1, 15 * 32, 15}; - cuda::std::array dst_strides{2236962 * 32, 32, 1}; - bench_copy(state, 0, shape, src_strides, 0, dst_strides); -} -NVBENCH_BENCH(transpose_src_small_15).set_name("transpose_src_small_15 (3D, int, 4GB)"); - -void transpose_src_small_16(nvbench::state& state) -{ - cuda::std::array shape{16, 2097152, 32}; - cuda::std::array src_strides{1, 16 * 32, 16}; - cuda::std::array dst_strides{2097152 * 32, 32, 1}; - bench_copy(state, 0, shape, src_strides, 0, dst_strides); -} -NVBENCH_BENCH(transpose_src_small_16).set_name("transpose_src_small_16 (3D, int, 4GB)"); - -void transpose_src_small_17(nvbench::state& state) -{ - cuda::std::array shape{17, 1973790, 32}; - cuda::std::array src_strides{1, 17 * 32, 17}; - cuda::std::array dst_strides{1973790 * 32, 32, 1}; - bench_copy(state, 0, shape, src_strides, 0, dst_strides); -} -NVBENCH_BENCH(transpose_src_small_17).set_name("transpose_src_small_17 (3D, int, 4GB)"); - -void transpose_dst_small_8_padded(nvbench::state& state) -{ - cuda::std::array shape{32, 4194304, 8}; - cuda::std::array src_strides{1, 32 * 8, 32}; - cuda::std::array dst_strides{4194304 * 16, 16, 1}; - bench_copy(state, 0, shape, src_strides, 0, dst_strides); -} -NVBENCH_BENCH(transpose_dst_small_8_padded).set_name("transpose_dst_small_8_padded (3D, int, 4GB)"); - -void transpose_dst_small_16_padded(nvbench::state& state) -{ - cuda::std::array shape{32, 2097152, 16}; - cuda::std::array src_strides{1, 32 * 16, 32}; - cuda::std::array dst_strides{2097152 * 32, 32, 1}; - bench_copy(state, 0, shape, src_strides, 0, dst_strides); -} -NVBENCH_BENCH(transpose_dst_small_16_padded).set_name("transpose_dst_small_16_padded (3D, int, 4GB)"); - -void transpose_src_small_16_4D(nvbench::state& state) -{ - cuda::std::array shape{16, 1024, 2048, 32}; - cuda::std::array src_strides{1, 16 * 32, 16 * 32 * 1024, 16}; - cuda::std::array dst_strides{1024 * 2048 * 32, 32, 1024 * 32, 1}; - bench_copy(state, 0, shape, src_strides, 0, dst_strides); -} -NVBENCH_BENCH(transpose_src_small_16_4D).set_name("transpose_src_small_16_4D (4D, int, 4GB)"); diff --git a/cccl_upstream/cudax/benchmarks/bench/cuco/common/defaults.cuh b/cccl_upstream/cudax/benchmarks/bench/cuco/common/defaults.cuh deleted file mode 100644 index 72fd6962..00000000 --- a/cccl_upstream/cudax/benchmarks/bench/cuco/common/defaults.cuh +++ /dev/null @@ -1,44 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include - -#include - -#include -#include - -namespace cuda::experimental::cuco::benchmark::defaults -{ -//! Key types covered by the default CUCO benchmark type axes. -using key_type_range = ::nvbench::type_list<::nvbench::int32_t, ::nvbench::int64_t>; -//! Value types covered by the default CUCO benchmark type axes. -using value_type_range = ::nvbench::type_list<::nvbench::int32_t, ::nvbench::int64_t>; - -//! Default number of inputs used when sweeping another benchmark axis. -inline constexpr auto n = ::nvbench::int64_t{100'000'000}; -//! Default fixed-capacity map target occupancy. -inline constexpr auto occupancy = 0.5; -//! Default lookup matching rate for contains-style benchmarks. -inline constexpr auto matching_rate = 1.0; -//! Default deterministic seed used by benchmark data generators. -inline constexpr auto seed = ::cuda::std::uint32_t{42}; - -//! Input-size sweep that remains cacheable for direct comparisons with CUCO benchmarks. -inline const auto n_range_cache = ::std::vector<::nvbench::int64_t>{8'000, 80'000, 800'000, 8'000'000, 80'000'000}; -//! Occupancy sweep used by fixed-capacity container benchmarks. -inline const auto occupancy_range = ::nvbench::range(0.1, 0.9, 0.1); -//! Average multiplicity sweep for duplicate-key distributions. -inline const auto multiplicity_range = ::std::vector{1.0, 2.0, 4.0, 8.0, 16.0}; -//! Matching-rate sweep used by contains-style benchmarks. -inline const auto matching_rate_range = ::nvbench::range(0.1, 1.0, 0.1); -} // namespace cuda::experimental::cuco::benchmark::defaults diff --git a/cccl_upstream/cudax/benchmarks/bench/cuco/common/key_generator.cuh b/cccl_upstream/cudax/benchmarks/bench/cuco/common/key_generator.cuh deleted file mode 100644 index a4601ac5..00000000 --- a/cccl_upstream/cudax/benchmarks/bench/cuco/common/key_generator.cuh +++ /dev/null @@ -1,285 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "defaults.cuh" -#include - -namespace cuda::experimental::cuco::benchmark -{ -namespace distribution -{ -//! Distribution tag for unique keys generated by shuffling the sequence `[0, N)`. -struct unique -{}; - -//! Distribution tag for uniformly sampled keys with controlled average multiplicity. -struct uniform -{ - //! Constructs a uniform distribution tag with the requested average key multiplicity. - //! - //! @param multiplicity Average number of generated keys that map to each unique key value. - //! @throws std::invalid_argument if `multiplicity` is not finite or is less than 1.0. - explicit uniform(double multiplicity) - : multiplicity{multiplicity} - { - if (!cuda::std::isfinite(multiplicity) || multiplicity < 1.0) - { - throw ::std::invalid_argument{"Multiplicity must be finite and at least 1"}; - } - } - - //! Average number of generated keys that map to each unique key value. - double multiplicity; -}; -} // namespace distribution - -namespace detail -{ -template -struct generate_uniform_fn -{ - __host__ __device__ constexpr generate_uniform_fn(cuda::std::size_t num, Distribution dist, cuda::std::size_t seed) - : num{num} - , dist{dist} - , seed{seed} - {} - - __host__ __device__ constexpr Key operator()(cuda::std::size_t idx) const noexcept - { - Rng rng; - rng.seed(seed + idx * 1664525ull + 1013904223ull); - - const auto num_unique_keys_unclamped = - static_cast(cuda::std::ceil(static_cast(num) / dist.multiplicity)); - const auto num_unique_keys = - num_unique_keys_unclamped < cuda::std::size_t{1} ? cuda::std::size_t{1} : num_unique_keys_unclamped; - thrust::uniform_int_distribution key_dist{Key{0}, static_cast(num_unique_keys - 1)}; - return key_dist(rng); - } - - cuda::std::size_t num; - Distribution dist; - cuda::std::size_t seed; -}; - -template -struct dropout_fn -{ - __host__ __device__ constexpr explicit dropout_fn(cuda::std::size_t num) - : num{num} - {} - - __host__ __device__ Key operator()(cuda::std::size_t seed) const noexcept - { - Rng rng; - thrust::uniform_int_distribution dist{static_cast(num), cuda::std::numeric_limits::max()}; - rng.seed(seed); - return dist(rng); - } - - cuda::std::size_t num; -}; - -template -struct dropout_pred -{ - __host__ __device__ constexpr explicit dropout_pred(double keep_prob) - : keep_prob{keep_prob} - {} - - __host__ __device__ bool operator()(cuda::std::size_t seed) const noexcept - { - Rng rng; - thrust::uniform_real_distribution dist{0.0, 1.0}; - rng.seed(seed); - return dist(rng) > keep_prob; - } - - double keep_prob; -}; -} // namespace detail - -//! Random key generator used by CUCO benchmarks. -//! -//! The generator defaults to `defaults::seed` to keep benchmark data reproducible across runs. -//! -//! @tparam Rng Pseudo-random number generator type compatible with Thrust random distributions. -template -class key_generator -{ -public: - //! Constructs a key generator with the given seed. - //! - //! @param seed Seed used to initialize the generator state. - explicit key_generator(cuda::std::uint32_t seed = defaults::seed) - : rng{seed} - {} - - //! Generates keys according to the given distribution using the default device execution policy. - //! - //! @tparam Distribution Distribution tag type. - //! @tparam OutputIt Output iterator type whose value type is the generated key type. - //! @param dist Distribution tag controlling how keys are generated. - //! @param out_begin Beginning of the output key range. - //! @param out_end End of the output key range. - //! @throws std::invalid_argument if `Distribution` is not a supported distribution tag. - template - void generate(Distribution dist, OutputIt out_begin, OutputIt out_end) - { - generate(dist, out_begin, out_end, thrust::device); - } - - //! Generates keys according to the given distribution using the provided execution policy. - //! - //! @tparam Distribution Distribution tag type. - //! @tparam OutputIt Output iterator type whose value type is the generated key type. - //! @tparam ExecPolicy Thrust execution policy type. - //! @param dist Distribution tag controlling how keys are generated. - //! @param out_begin Beginning of the output key range. - //! @param out_end End of the output key range. - //! @param exec_policy Execution policy used for the underlying Thrust algorithms. - //! @throws std::invalid_argument if `Distribution` is not a supported distribution tag. - template - void generate(Distribution dist, OutputIt out_begin, OutputIt out_end, ExecPolicy exec_policy) - { - using value_type = typename cuda::std::iterator_traits::value_type; - - if constexpr (cuda::std::is_same_v) - { - thrust::sequence(exec_policy, out_begin, out_end, value_type{0}); - thrust::shuffle(exec_policy, out_begin, out_end, rng); - } - else if constexpr (cuda::std::is_same_v) - { - const auto num_keys = static_cast(cuda::std::distance(out_begin, out_end)); - const auto seed = static_cast(rng()); - - thrust::transform( - exec_policy, - cuda::counting_iterator{0}, - cuda::counting_iterator{num_keys}, - out_begin, - detail::generate_uniform_fn{num_keys, dist, seed}); - } - else - { - throw ::std::invalid_argument{"Unexpected distribution type"}; - } - } - - //! Drops keys with probability `1 - keep_prob` using the default device execution policy. - //! - //! Replaced keys are sampled from `[N, max_key]`, where `N` is the number of keys in the range. - //! - //! The full range is shuffled afterward, even when all keys are kept. - //! - //! @tparam InOutIt Mutable iterator type whose value type is the key type. - //! @param begin Beginning of the key range to update in place. - //! @param end End of the key range to update in place. - //! @param keep_prob Probability of keeping each original key. Must be in `[0, 1]`. - //! @throws std::invalid_argument if `keep_prob` is outside `[0, 1]`. - template - void dropout(InOutIt begin, InOutIt end, double keep_prob) - { - dropout(begin, end, keep_prob, thrust::device); - } - - //! Drops keys with probability `1 - keep_prob` using the provided execution policy. - //! - //! Replaced keys are sampled from `[N, max_key]`, where `N` is the number of keys in the range. - //! - //! The full range is shuffled afterward, even when all keys are kept. - //! - //! @tparam InOutIt Mutable iterator type whose value type is the key type. - //! @tparam ExecPolicy Thrust execution policy type. - //! @param begin Beginning of the key range to update in place. - //! @param end End of the key range to update in place. - //! @param keep_prob Probability of keeping each original key. Must be in `[0, 1]`. - //! @param exec_policy Execution policy used for the underlying Thrust algorithms. - //! @throws std::invalid_argument if `keep_prob` is outside `[0, 1]`. - template - void dropout(InOutIt begin, InOutIt end, double keep_prob, ExecPolicy exec_policy) - { - using value_type = typename cuda::std::iterator_traits::value_type; - - if (keep_prob < 0.0 || keep_prob > 1.0) - { - throw ::std::invalid_argument{"Probability needs to be between 0 and 1"}; - } - - if (keep_prob < 1.0) - { - const auto num_keys = static_cast(cuda::std::distance(begin, end)); - cuda::counting_iterator seeds{static_cast(rng())}; - - thrust::transform_if( - exec_policy, - seeds, - seeds + num_keys, - begin, - detail::dropout_fn{num_keys}, - detail::dropout_pred{keep_prob}); - } - - thrust::shuffle(exec_policy, begin, end, rng); - } - -private: - Rng rng; -}; - -//! Constructs the requested distribution tag from NVBench axis values. -//! -//! `distribution::uniform` reads the `Multiplicity` axis from `state`. -//! -//! @tparam Distribution Distribution tag type to construct. -//! @param state NVBench state containing distribution-specific axis values. -//! @return Distribution tag initialized from the benchmark state. -//! @throws std::invalid_argument if `Distribution` is not a supported distribution tag. -template -Distribution dist_from_state(nvbench::state const& state) -{ - if constexpr (cuda::std::is_same_v) - { - return Distribution{}; - } - else if constexpr (cuda::std::is_same_v) - { - return Distribution{state.get_float64("Multiplicity")}; - } - else - { - throw ::std::invalid_argument{"Unexpected distribution type"}; - } -} -} // namespace cuda::experimental::cuco::benchmark - -NVBENCH_DECLARE_TYPE_STRINGS( - cuda::experimental::cuco::benchmark::distribution::unique, "UNIQUE", "distribution::unique"); -NVBENCH_DECLARE_TYPE_STRINGS( - cuda::experimental::cuco::benchmark::distribution::uniform, "UNIFORM", "distribution::uniform"); diff --git a/cccl_upstream/cudax/benchmarks/bench/cuco/fixed_capacity_map/contains.cu b/cccl_upstream/cudax/benchmarks/bench/cuco/fixed_capacity_map/contains.cu deleted file mode 100644 index b3453504..00000000 --- a/cccl_upstream/cudax/benchmarks/bench/cuco/fixed_capacity_map/contains.cu +++ /dev/null @@ -1,110 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "../common/defaults.cuh" -#include "../common/key_generator.cuh" -#include - -namespace cudax = cuda::experimental; -namespace bench = cudax::cuco::benchmark; - -/** - * @brief A benchmark evaluating `cudax::cuco::fixed_capacity_map::contains_async` performance. - */ -template -void fixed_capacity_map_contains(nvbench::state& state, nvbench::type_list) -{ - if constexpr (sizeof(Key) != sizeof(Value)) - { - state.skip("Key and Value must have the same size."); - } - else - { - using pair_type = cuda::std::pair; - using map_type = cudax::cuco::fixed_capacity_map; - - const auto num_keys = state.get_int64("NumInputs"); - const auto occupancy = state.get_float64("Occupancy"); - const auto matching_rate = state.get_float64("MatchingRate"); - - const auto size = static_cast(static_cast(num_keys) / occupancy); - - const auto device = cuda::device_ref{0}; - cuda::stream stream{device}; - const cuda::device_memory_pool_ref mr = cuda::device_default_memory_pool(device); - const auto exec_policy = thrust::cuda::par_nosync.on(stream.get()); - - auto keys = cuda::make_device_buffer(stream, device, num_keys, cuda::no_init); - - bench::key_generator gen{}; - gen.generate(bench::dist_from_state(state), keys.begin(), keys.end(), exec_policy); - - auto pairs = cuda::make_device_buffer(stream, device, num_keys, cuda::no_init); - thrust::transform(exec_policy, keys.begin(), keys.end(), pairs.begin(), [] __device__(Key const& key) { - return pair_type{key, Value{}}; - }); - - map_type map{stream, mr, size, cudax::cuco::empty_key(Key{-1}), cudax::cuco::empty_value(Value{-1})}; - map.insert(stream, pairs.begin(), pairs.end()); - - gen.dropout(keys.begin(), keys.end(), matching_rate, exec_policy); - - auto result = cuda::make_device_buffer(stream, device, num_keys, cuda::no_init); - stream.sync(); - - state.add_element_count(num_keys); - state.exec([&](nvbench::launch& launch) { - map.contains_async({launch.get_stream()}, keys.begin(), keys.end(), result.begin()); - }); - } -} - -NVBENCH_BENCH_TYPES(fixed_capacity_map_contains, - NVBENCH_TYPE_AXES(bench::defaults::key_type_range, - bench::defaults::value_type_range, - nvbench::type_list)) - .set_name("fixed_capacity_map_contains_unique_capacity") - .set_type_axes_names({"Key", "Value", "Distribution"}) - .add_int64_axis("NumInputs", bench::defaults::n_range_cache) - .add_float64_axis("Occupancy", {bench::defaults::occupancy}) - .add_float64_axis("MatchingRate", {bench::defaults::matching_rate}); - -NVBENCH_BENCH_TYPES(fixed_capacity_map_contains, - NVBENCH_TYPE_AXES(bench::defaults::key_type_range, - bench::defaults::value_type_range, - nvbench::type_list)) - .set_name("fixed_capacity_map_contains_unique_occupancy") - .set_type_axes_names({"Key", "Value", "Distribution"}) - .add_int64_axis("NumInputs", {bench::defaults::n}) - .add_float64_axis("Occupancy", bench::defaults::occupancy_range) - .add_float64_axis("MatchingRate", {bench::defaults::matching_rate}); - -NVBENCH_BENCH_TYPES(fixed_capacity_map_contains, - NVBENCH_TYPE_AXES(bench::defaults::key_type_range, - bench::defaults::value_type_range, - nvbench::type_list)) - .set_name("fixed_capacity_map_contains_unique_matching_rate") - .set_type_axes_names({"Key", "Value", "Distribution"}) - .add_int64_axis("NumInputs", {bench::defaults::n}) - .add_float64_axis("Occupancy", {bench::defaults::occupancy}) - .add_float64_axis("MatchingRate", bench::defaults::matching_rate_range); diff --git a/cccl_upstream/cudax/benchmarks/bench/cuco/fixed_capacity_map/find.cu b/cccl_upstream/cudax/benchmarks/bench/cuco/fixed_capacity_map/find.cu deleted file mode 100644 index e99381b7..00000000 --- a/cccl_upstream/cudax/benchmarks/bench/cuco/fixed_capacity_map/find.cu +++ /dev/null @@ -1,108 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "../common/defaults.cuh" -#include "../common/key_generator.cuh" -#include - -namespace cudax = cuda::experimental; -namespace bench = cudax::cuco::benchmark; - -/** - * @brief A benchmark evaluating `cudax::cuco::fixed_capacity_map::find_async` performance. - */ -template -void fixed_capacity_map_find(nvbench::state& state, nvbench::type_list) -{ - if constexpr (sizeof(Key) != sizeof(Value)) - { - state.skip("Key and Value must have the same size."); - } - else - { - using pair_type = cuda::std::pair; - using map_type = cudax::cuco::fixed_capacity_map; - - const auto num_keys = static_cast<::cuda::std::size_t>(state.get_int64("NumInputs")); - const auto occupancy = state.get_float64("Occupancy"); - const auto matching_rate = state.get_float64("MatchingRate"); - - const auto device = cuda::device_ref{0}; - cuda::stream stream{device}; - const cuda::device_memory_pool_ref mr = cuda::device_default_memory_pool(device); - const auto exec_policy = thrust::cuda::par_nosync.on(stream.get()); - - auto keys = cuda::make_device_buffer(stream, device, num_keys, cuda::no_init); - - bench::key_generator gen{}; - gen.generate(bench::dist_from_state(state), keys.begin(), keys.end(), exec_policy); - - auto pairs = cuda::make_device_buffer(stream, device, num_keys, cuda::no_init); - thrust::transform(exec_policy, keys.begin(), keys.end(), pairs.begin(), [] __device__(Key const& key) { - return pair_type{key, Value{}}; - }); - - map_type map{stream, mr, num_keys, occupancy, cudax::cuco::empty_key(Key{-1}), cudax::cuco::empty_value(Value{-1})}; - map.insert(stream, pairs.begin(), pairs.end()); - - gen.dropout(keys.begin(), keys.end(), matching_rate, exec_policy); - - auto result = cuda::make_device_buffer(stream, device, num_keys, cuda::no_init); - stream.sync(); - - state.add_element_count(num_keys); - state.exec([&](nvbench::launch& launch) { - map.find_async({launch.get_stream()}, keys.begin(), keys.end(), result.begin()); - }); - } -} - -NVBENCH_BENCH_TYPES(fixed_capacity_map_find, - NVBENCH_TYPE_AXES(bench::defaults::key_type_range, - bench::defaults::value_type_range, - nvbench::type_list)) - .set_name("fixed_capacity_map_find_unique_capacity") - .set_type_axes_names({"Key", "Value", "Distribution"}) - .add_int64_axis("NumInputs", bench::defaults::n_range_cache) - .add_float64_axis("Occupancy", {bench::defaults::occupancy}) - .add_float64_axis("MatchingRate", {bench::defaults::matching_rate}); - -NVBENCH_BENCH_TYPES(fixed_capacity_map_find, - NVBENCH_TYPE_AXES(bench::defaults::key_type_range, - bench::defaults::value_type_range, - nvbench::type_list)) - .set_name("fixed_capacity_map_find_unique_occupancy") - .set_type_axes_names({"Key", "Value", "Distribution"}) - .add_int64_axis("NumInputs", {bench::defaults::n}) - .add_float64_axis("Occupancy", bench::defaults::occupancy_range) - .add_float64_axis("MatchingRate", {bench::defaults::matching_rate}); - -NVBENCH_BENCH_TYPES(fixed_capacity_map_find, - NVBENCH_TYPE_AXES(bench::defaults::key_type_range, - bench::defaults::value_type_range, - nvbench::type_list)) - .set_name("fixed_capacity_map_find_unique_matching_rate") - .set_type_axes_names({"Key", "Value", "Distribution"}) - .add_int64_axis("NumInputs", {bench::defaults::n}) - .add_float64_axis("Occupancy", {bench::defaults::occupancy}) - .add_float64_axis("MatchingRate", bench::defaults::matching_rate_range); diff --git a/cccl_upstream/cudax/benchmarks/bench/cuco/fixed_capacity_map/insert.cu b/cccl_upstream/cudax/benchmarks/bench/cuco/fixed_capacity_map/insert.cu deleted file mode 100644 index 607e914e..00000000 --- a/cccl_upstream/cudax/benchmarks/bench/cuco/fixed_capacity_map/insert.cu +++ /dev/null @@ -1,105 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "../common/defaults.cuh" -#include "../common/key_generator.cuh" -#include - -namespace cudax = cuda::experimental; -namespace bench = cudax::cuco::benchmark; - -/** - * @brief A benchmark evaluating `cudax::cuco::fixed_capacity_map::insert_async` performance. - */ -template -void fixed_capacity_map_insert(nvbench::state& state, nvbench::type_list) -{ - if constexpr (sizeof(Key) != sizeof(Value)) - { - state.skip("Key and Value must have the same size."); - } - else - { - using pair_type = cuda::std::pair; - using map_type = cudax::cuco::fixed_capacity_map; - - const auto num_keys = state.get_int64("NumInputs"); - const auto occupancy = state.get_float64("Occupancy"); - - const auto size = static_cast(static_cast(num_keys) / occupancy); - - const auto device = cuda::device_ref{0}; - cuda::stream stream{device}; - const cuda::device_memory_pool_ref mr = cuda::device_default_memory_pool(device); - const auto exec_policy = thrust::cuda::par_nosync.on(stream.get()); - - auto keys = cuda::make_device_buffer(stream, device, num_keys, cuda::no_init); - - bench::key_generator gen{}; - gen.generate(bench::dist_from_state(state), keys.begin(), keys.end(), exec_policy); - - auto pairs = cuda::make_device_buffer(stream, device, num_keys, cuda::no_init); - thrust::transform(exec_policy, keys.begin(), keys.end(), pairs.begin(), [] __device__(Key const& key) { - return pair_type{key, Value{}}; - }); - - map_type map{stream, mr, size, cudax::cuco::empty_key(Key{-1}), cudax::cuco::empty_value(Value{-1})}; - stream.sync(); - - state.add_element_count(num_keys); - state.exec(nvbench::exec_tag::timer, [&](nvbench::launch& launch, auto& timer) { - timer.start(); - map.insert_async({launch.get_stream()}, pairs.begin(), pairs.end()); - timer.stop(); - map.clear_async({launch.get_stream()}); - }); - } -} - -NVBENCH_BENCH_TYPES(fixed_capacity_map_insert, - NVBENCH_TYPE_AXES(bench::defaults::key_type_range, - bench::defaults::value_type_range, - nvbench::type_list)) - .set_name("fixed_capacity_map_insert_unique_capacity") - .set_type_axes_names({"Key", "Value", "Distribution"}) - .add_int64_axis("NumInputs", bench::defaults::n_range_cache) - .add_float64_axis("Occupancy", {bench::defaults::occupancy}); - -NVBENCH_BENCH_TYPES(fixed_capacity_map_insert, - NVBENCH_TYPE_AXES(bench::defaults::key_type_range, - bench::defaults::value_type_range, - nvbench::type_list)) - .set_name("fixed_capacity_map_insert_unique_occupancy") - .set_type_axes_names({"Key", "Value", "Distribution"}) - .add_int64_axis("NumInputs", {bench::defaults::n}) - .add_float64_axis("Occupancy", bench::defaults::occupancy_range); - -NVBENCH_BENCH_TYPES(fixed_capacity_map_insert, - NVBENCH_TYPE_AXES(bench::defaults::key_type_range, - bench::defaults::value_type_range, - nvbench::type_list)) - .set_name("fixed_capacity_map_insert_uniform_multiplicity") - .set_type_axes_names({"Key", "Value", "Distribution"}) - .add_int64_axis("NumInputs", {bench::defaults::n}) - .add_float64_axis("Occupancy", {bench::defaults::occupancy}) - .add_float64_axis("Multiplicity", bench::defaults::multiplicity_range); diff --git a/cccl_upstream/cudax/benchmarks/bench/cuco/hashers.cu b/cccl_upstream/cudax/benchmarks/bench/cuco/hashers.cu deleted file mode 100644 index 291b5b77..00000000 --- a/cccl_upstream/cudax/benchmarks/bench/cuco/hashers.cu +++ /dev/null @@ -1,131 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -// SPDX-License-Identifier: SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -#include - -#include -#include - -#include - -#include -#include - -namespace cudax = cuda::experimental; - -// repeat hash computation n times -static constexpr auto n_repeats = 100; - -template -struct large_key -{ - constexpr __host__ __device__ large_key(cuda::std::int32_t seed) noexcept - { - for (cuda::std::int32_t i = 0; i < Words; ++i) - { - data_[i] = seed; - } - } - -private: - cuda::std::int32_t data_[Words]; -}; - -template -__global__ void hash_bench_kernel(Hasher hash, size_t n, OutputIt out, bool materialize_result) -{ - size_t const gid = static_cast(BlockSize) * blockIdx.x + threadIdx.x; - size_t const loop_stride = static_cast(gridDim.x) * BlockSize; - size_t idx = gid; - using result_t = decltype(hash(0)); - - result_t agg{}; - - while (idx < n) - { - Key key(idx); - for (cuda::std::int32_t i = 0; i < n_repeats; ++i) - { // execute hash func n times - agg += hash(key); - } - idx += loop_stride; - } - - if (materialize_result) - { - out[gid] = agg; - } -} - -// benchmark evaluating performance of various hash functions -template -void hash_eval(nvbench::state& state, nvbench::type_list) -{ - using Hash = typename HasherTag::template fn; - - bool const materialize_result = false; - constexpr auto block_size = 128; - auto const num_keys = state.get_int64("NumInputs"); - auto const grid_size = (num_keys + block_size * 16 - 1) / block_size * 16; - using result_t = decltype(std::declval()(std::declval())); - - thrust::device_vector hash_values((materialize_result) ? num_keys : 1); - - state.add_element_count(num_keys); - - state.exec([&](nvbench::launch& launch) { - hash_bench_kernel - <<>>(Hash{}, num_keys, hash_values.begin(), materialize_result); - }); -} - -struct xxhash_32_tag -{ - template - using fn = cudax::cuco::hash; -}; - -struct xxhash_64_tag -{ - template - using fn = cudax::cuco::hash; -}; - -struct murmurhash3_32_tag -{ - template - using fn = cudax::cuco::hash; -}; - -#if _CCCL_HAS_INT128() - -struct murmurhash3_x86_128_tag -{ - template - using fn = cudax::cuco::hash; -}; - -struct murmurhash3_x64_128_tag -{ - template - using fn = cudax::cuco::hash; -}; - -#endif // _CCCL_HAS_INT128() - -NVBENCH_BENCH_TYPES( - hash_eval, - NVBENCH_TYPE_AXES( - nvbench::type_list, - nvbench::type_list, large_key<8>, large_key<16>, large_key<32>>)) - .set_name("hash_function_eval") - .set_type_axes_names({"Hash", "Key"}) - .add_int64_power_of_two_axis("NumInputs", nvbench::range(18, 26, 4)); diff --git a/cccl_upstream/cudax/benchmarks/bench/cuco/hyperloglog.cu b/cccl_upstream/cudax/benchmarks/bench/cuco/hyperloglog.cu deleted file mode 100644 index 175fc6b5..00000000 --- a/cccl_upstream/cudax/benchmarks/bench/cuco/hyperloglog.cu +++ /dev/null @@ -1,133 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#include -#include - -#include -#include -#include -#include -#include - -#include - -#include "common/defaults.cuh" -#include - -namespace cudax = cuda::experimental; -namespace bench = cudax::cuco::benchmark; - -namespace -{ -template -void add_relative_error_summary( - nvbench::state& state, - cudax::cuco::hyperloglog& estimator, - cuda::stream_ref stream, - Key* first, - cuda::std::size_t num_items) -{ - estimator.add(stream, first, first + num_items); - const auto estimated_cardinality = estimator.estimate(stream); - const auto relative_error = - cuda::std::abs(static_cast(estimated_cardinality) / static_cast(num_items) - 1.0); - estimator.clear(stream); - - auto& summary = state.add_summary("RelativeError"); - summary.set_string("hint", "RelErr"); - summary.set_string("short_name", "RelativeError"); - summary.set_string("description", "Relative approximation error."); - summary.set_float64("value", relative_error); -} -} // namespace - -/** - * @brief A benchmark evaluating `cudax::cuco::hyperloglog` end-to-end performance. - */ -template -void hyperloglog_e2e(nvbench::state& state, nvbench::type_list) -{ - using estimator_type = cudax::cuco::hyperloglog; - using sketch_size_kb_type = typename estimator_type::sketch_size_kb; - - const auto num_items = static_cast(state.get_int64("NumInputs")); - const auto sketch_size_kb = sketch_size_kb_type{static_cast(state.get_int64("SketchSizeKB"))}; - - const auto device = cuda::device_ref{0}; - cuda::stream stream{device}; - const cuda::device_memory_pool_ref mr = cuda::device_default_memory_pool(device); - - auto items = cuda::make_device_buffer(stream, device, num_items, cuda::no_init); - thrust::sequence(thrust::cuda::par_nosync.on(stream.get()), items.begin(), items.end(), Key{0}); - - estimator_type estimator{stream, mr, sketch_size_kb}; - stream.sync(); - - state.add_element_count(num_items); - state.add_global_memory_reads(num_items, "InputSize"); - - add_relative_error_summary(state, estimator, stream, items.data(), num_items); - - state.exec(nvbench::exec_tag::sync | nvbench::exec_tag::timer, [&](nvbench::launch& launch, auto& timer) { - timer.start(); - estimator.add_async({launch.get_stream()}, items.begin(), items.end()); - [[maybe_unused]] const auto estimated_cardinality = estimator.estimate({launch.get_stream()}); - timer.stop(); - - estimator.clear_async({launch.get_stream()}); - }); -} - -/** - * @brief A benchmark evaluating `cudax::cuco::hyperloglog::add_async` performance. - */ -template -void hyperloglog_add(nvbench::state& state, nvbench::type_list) -{ - using estimator_type = cudax::cuco::hyperloglog; - using sketch_size_kb_type = typename estimator_type::sketch_size_kb; - - const auto num_items = static_cast(state.get_int64("NumInputs")); - const auto sketch_size_kb = sketch_size_kb_type{static_cast(state.get_int64("SketchSizeKB"))}; - - const auto device = cuda::device_ref{0}; - cuda::stream stream{device}; - const cuda::device_memory_pool_ref mr = cuda::device_default_memory_pool(device); - - auto items = cuda::make_device_buffer(stream, device, num_items, cuda::no_init); - thrust::sequence(thrust::cuda::par_nosync.on(stream.get()), items.begin(), items.end(), Key{0}); - - estimator_type estimator{stream, mr, sketch_size_kb}; - stream.sync(); - - state.add_element_count(num_items); - state.add_global_memory_reads(num_items, "InputSize"); - - state.exec(nvbench::exec_tag::timer, [&](nvbench::launch& launch, auto& timer) { - timer.start(); - estimator.add_async({launch.get_stream()}, items.begin(), items.end()); - timer.stop(); - - estimator.clear_async({launch.get_stream()}); - }); -} - -NVBENCH_BENCH_TYPES(hyperloglog_e2e, NVBENCH_TYPE_AXES(bench::defaults::key_type_range)) - .set_name("hyperloglog_e2e") - .set_type_axes_names({"Key"}) - .add_int64_power_of_two_axis("NumInputs", {30}) - .add_int64_axis("SketchSizeKB", {8, 16, 32, 64, 128, 256}); - -NVBENCH_BENCH_TYPES(hyperloglog_add, NVBENCH_TYPE_AXES(bench::defaults::key_type_range)) - .set_name("hyperloglog_add") - .set_type_axes_names({"Key"}) - .add_int64_power_of_two_axis("NumInputs", {30}) - .add_int64_axis("SketchSizeKB", {8, 16, 32, 64, 128, 256}); diff --git a/cccl_upstream/cudax/cmake/cudaxAddSubdir.cmake b/cccl_upstream/cudax/cmake/cudaxAddSubdir.cmake deleted file mode 100644 index bae94045..00000000 --- a/cccl_upstream/cudax/cmake/cudaxAddSubdir.cmake +++ /dev/null @@ -1 +0,0 @@ -cccl_add_subdir_helper(cudax) diff --git a/cccl_upstream/cudax/cmake/cudaxBuildCompilerTargets.cmake b/cccl_upstream/cudax/cmake/cudaxBuildCompilerTargets.cmake deleted file mode 100644 index 87622408..00000000 --- a/cccl_upstream/cudax/cmake/cudaxBuildCompilerTargets.cmake +++ /dev/null @@ -1,65 +0,0 @@ -# Including this file defines the following targets: -# -# cudax.compiler_interface -# - Interface target that includes all compiler settings for cudax tests, etc. - -cccl_get_cub() -cccl_get_cudax() -cccl_get_libcudacxx() -cccl_get_thrust() - -set(cuda_compile_options) -set(cxx_compile_options) -set(cxx_compile_definitions) - -if ("MSVC" STREQUAL "${CMAKE_CXX_COMPILER_ID}") - # C4848: support for attribute 'msvc::no_unique_address' in C++17 and earlier is a vendor extension - append_option_if_available("/wd4848" cxx_compile_options) - - # XXX Temporary hack for STF ! - # C4267: conversion from 'meow' to 'purr', possible loss of data - append_option_if_available("/wd4267" cxx_compile_options) - - # C4459 : declaration of 'identifier' hides global declaration - # We work around std::chrono::last which hides some internal "last" variable - append_option_if_available("/wd4459" cxx_compile_options) - - # stf used getenv which is potentially unsafe but not in our context - list(APPEND cxx_compile_definitions "_CRT_SECURE_NO_WARNINGS") -endif() - -if ("Clang" STREQUAL "${CMAKE_CXX_COMPILER_ID}") - # stf heavily uses host device lambdas which break on clang due to a warning about the implicitly - # deleted copy constructor - # TODO(bgruber): remove this when NVBug 4980157 is resolved - append_option_if_available("-Wno-deprecated-copy" cxx_compile_options) -endif() - -list(APPEND cxx_compile_definitions CCCL_ENABLE_ASSERTIONS) - -# Some groups related experimental code is located directly in libcu++ and is guarded by -# _CUDAX_ENABLE_GROUP_FEATURES_IN_LIBCUDACXX macro, otherwise it would lead to a lot of code duplication. We define this -# macro for cudax code globally, to get access to get access to the code. -# -# Can be removed once groups are no longer experimental. -list(APPEND cxx_compile_definitions _CUDAX_ENABLE_GROUP_FEATURES_IN_LIBCUDACXX) - -cccl_build_compiler_interface( - cudax.compiler_flags - "${cuda_compile_options}" - "${cxx_compile_options}" - "${cxx_compile_definitions}" -) - -add_library(cudax.compiler_interface INTERFACE) -target_link_libraries( - cudax.compiler_interface - INTERFACE - # order matters here, we need the cudax options to override the cccl options. - cccl.compiler_interface - cudax.compiler_flags - libcudacxx::libcudacxx - CUB::CUB - Thrust::Thrust - cudax::cudax -) diff --git a/cccl_upstream/cudax/cmake/cudaxHeaderTesting.cmake b/cccl_upstream/cudax/cmake/cudaxHeaderTesting.cmake deleted file mode 100644 index 88946f84..00000000 --- a/cccl_upstream/cudax/cmake/cudaxHeaderTesting.cmake +++ /dev/null @@ -1,109 +0,0 @@ -# For every public header, build a translation unit containing `#include
` -# to let the compiler try to figure out warnings in that header if it is not otherwise -# included in tests, and also to verify if the headers are modular enough. -# .inl files are not globbed for, because they are not supposed to be used as public -# entrypoints. - -cccl_get_cudatoolkit() - -# Meta target for all configs' header builds: -add_custom_target(cudax.all.headers) - -function(cudax_add_header_test label definitions) - ################### - # Non-STF headers # - set(headertest_target cudax.headers.${label}.no_stf) - cccl_generate_header_tests( - ${headertest_target} - cudax/include - # The cudax header template removes the check for the `small` macro. - HEADER_TEMPLATE "${cudax_SOURCE_DIR}/cmake/header_test.in.cu" - GLOBS "cuda/experimental/*.cuh" - EXCLUDES - # The following internal headers are not required to compile independently: - "cuda/experimental/__execution/prologue.cuh" - "cuda/experimental/__execution/epilogue.cuh" - # cuFile headers are compiled separately: - "cuda/experimental/cufile.cuh" - "cuda/experimental/__cufile/*" - # Places headers are compiled separately: - "cuda/experimental/places.cuh" - "cuda/experimental/__places/*" - # STF headers are compiled separately: - "cuda/experimental/stf.cuh" - "cuda/experimental/__stf/*" - ) - target_link_libraries(${headertest_target} PUBLIC cudax.compiler_interface) - - if (cudax_ENABLE_CUFILE) - ############### - # cuFile headers # - set(headertest_target cudax.headers.${label}.cufile) - cccl_generate_header_tests( - ${headertest_target} - cudax/include - HEADER_TEMPLATE "${cudax_SOURCE_DIR}/cmake/header_test.in.cu" - GLOBS # - "cuda/experimental/cufile.cuh" - "cuda/experimental/__cufile/*.cuh" - ) - target_link_libraries(${headertest_target} PUBLIC cudax.compiler_interface) - endif() - - # FIXME: Enable MSVC - if (cudax_ENABLE_PLACES AND NOT "MSVC" STREQUAL "${CMAKE_CXX_COMPILER_ID}") - ################## - # Places headers # - set(headertest_target cudax.headers.${label}.places) - cccl_generate_header_tests( - ${headertest_target} - cudax/include - GLOBS # - "cuda/experimental/places.cuh" - "cuda/experimental/__places/*.cuh" - HEADER_TEMPLATE "${cudax_SOURCE_DIR}/cmake/header_test.in.cu" - ) - target_link_libraries(${headertest_target} PUBLIC cudax.compiler_interface) - target_compile_options( - ${headertest_target} - PRIVATE - $<$:--extended-lambda> - $<$:--expt-relaxed-constexpr> - ) - endif() - - # FIXME: Enable MSVC - if (cudax_ENABLE_CUDASTF AND NOT "MSVC" STREQUAL "${CMAKE_CXX_COMPILER_ID}") - ############### - # STF headers # - set(headertest_target cudax.headers.${label}.stf) - cccl_generate_header_tests( - ${headertest_target} - cudax/include - GLOBS # - "cuda/experimental/stf.cuh" - "cuda/experimental/__stf/*.cuh" - # FIXME: The cudax header template removes the check for the `small` macro. - # cuda/experimental/__stf/utility/memory.cuh defines functions named `small`. - # These should be renamed to avoid conflicts with windows system headers, and - # the following line removed: - HEADER_TEMPLATE "${cudax_SOURCE_DIR}/cmake/header_test.in.cu" - ) - target_link_libraries( - ${headertest_target} - PUBLIC cudax.compiler_interface CUDA::cuda_driver - ) - target_compile_options( - ${headertest_target} - PRIVATE - # Required by stf headers: - $<$:--extended-lambda> - # FIXME: We should be able to refactor away from needing this by - # using _CCCL_HOST_DEVICE and friends + `::cuda::std` utilities where - # necessary. - $<$:--expt-relaxed-constexpr> - ) - endif() -endfunction() - -cudax_add_header_test(basic "") diff --git a/cccl_upstream/cudax/cmake/cudaxPlacesConfigureTarget.cmake b/cccl_upstream/cudax/cmake/cudaxPlacesConfigureTarget.cmake deleted file mode 100644 index af57ce4d..00000000 --- a/cccl_upstream/cudax/cmake/cudaxPlacesConfigureTarget.cmake +++ /dev/null @@ -1,23 +0,0 @@ -# Configures a target for the Places framework. -function(cudax_places_configure_target target_name) - target_link_libraries( - ${target_name} - PRIVATE # - CUDA::cudart_static - CUDA::cuda_driver - ) - - target_compile_options( - ${target_name} - PRIVATE - $<$:--extended-lambda> - $<$:--expt-relaxed-constexpr> - ) - - set_target_properties( - ${target_name} - PROPERTIES # - CUDA_RUNTIME_LIBRARY Static - CUDA_SEPARABLE_COMPILATION ON - ) -endfunction() diff --git a/cccl_upstream/cudax/cmake/cudaxSTFConfigureTarget.cmake b/cccl_upstream/cudax/cmake/cudaxSTFConfigureTarget.cmake deleted file mode 100644 index d12cfbfe..00000000 --- a/cccl_upstream/cudax/cmake/cudaxSTFConfigureTarget.cmake +++ /dev/null @@ -1,58 +0,0 @@ -# Configures a target for the STF framework. -function(cudax_stf_configure_target target_name) - set(options LINK_MATHLIBS) - set(oneValueArgs) - set(multiValueArgs) - cmake_parse_arguments( - CSCT - "${options}" - "${oneValueArgs}" - "${multiValueArgs}" - ${ARGN} - ) - - target_link_libraries( - ${target_name} - PRIVATE # - CUDA::cudart_static - CUDA::curand - CUDA::cuda_driver - ) - - if (cudax_ENABLE_CUDASTF_CODE_GENERATION) - target_compile_options( - ${target_name} - PRIVATE $<$:--extended-lambda> - ) - else() - target_compile_definitions( - ${target_name} - PRIVATE "CUDASTF_DISABLE_CODE_GENERATION" - ) - endif() - - target_compile_options( - ${target_name} - PRIVATE $<$:--expt-relaxed-constexpr> - ) - - set_target_properties( - ${target_name} - PROPERTIES # - CUDA_RUNTIME_LIBRARY Static - CUDA_SEPARABLE_COMPILATION ON - ) - - if (CSCT_LINK_MATHLIBS) - target_link_libraries( - ${target_name} - PRIVATE # - CUDA::cublas - CUDA::cusolver - ) - endif() - - if (cudax_ENABLE_CUDASTF_BOUNDSCHECK) - target_compile_definitions(${target_name} PRIVATE "CUDASTF_BOUNDSCHECK") - endif() -endfunction() diff --git a/cccl_upstream/cudax/cmake/header_test.in.cu b/cccl_upstream/cudax/cmake/header_test.in.cu deleted file mode 100644 index 1e31b2e6..00000000 --- a/cccl_upstream/cudax/cmake/header_test.in.cu +++ /dev/null @@ -1,66 +0,0 @@ -// This source file checks that: -// 1) Header <@header@> compiles without error. -// 2) Common macro collisions with platform/system headers are avoided. -// 3) half/bf16 aren't included when these are explicitly disabled. - -// Define CUDAX_MACRO_CHECK(macro, header), which emits a diagnostic indicating -// a potential macro collision and halts. -// -// Use raw platform checks instead of the CCCL macros since we -// don't want to #include any headers other than the one being tested. -// -// This is only implemented for MSVC/GCC/Clang. -#if defined(_MSC_VER) // MSVC - -// Fake up an error for MSVC -# define CUDAX_MACRO_CHECK_IMPL(msg) \ - /* Print message that looks like an error: */ \ - __pragma(message(__FILE__ ":" CUDAX_MACRO_CHECK_IMPL0(__LINE__) ": error: " #msg)) static_assert(false, #msg); -# define CUDAX_MACRO_CHECK_IMPL0(x) CUDAX_MACRO_CHECK_IMPL1(x) -# define CUDAX_MACRO_CHECK_IMPL1(x) #x - -#elif defined(__clang__) || defined(__GNUC__) - -// GCC/clang are easy: -# define CUDAX_MACRO_CHECK_IMPL(msg) CUDAX_MACRO_CHECK_IMPL0(GCC error #msg) -# define CUDAX_MACRO_CHECK_IMPL0(expr) _Pragma(#expr) - -#endif - -// Hacky way to build a string, but it works on all tested platforms. -#define CUDAX_MACRO_CHECK(MACRO, HEADER) \ - CUDAX_MACRO_CHECK_IMPL(Identifier MACRO should not be used from CCCL headers due to conflicts with HEADER macros.) - -// complex.h conflicts -#define I CUDAX_MACRO_CHECK('I', complex.h) - -// windows.h conflicts -// @eniebler 2024-08-30: This test is disabled because it causes build -// failures in some configurations. -// #define small CUDAX_MACRO_CHECK('small', windows.h) -// We can't enable these checks without breaking some builds -- some standard -// library implementations unconditionally `#undef` these macros, which then -// causes random failures later. -// Leaving these commented out as a warning: Here be dragons. -// #define min(...) CUDAX_MACRO_CHECK('min', windows.h) -// #define max(...) CUDAX_MACRO_CHECK('max', windows.h) - -// termios.h conflicts (NVIDIA/thrust#1547) -#define B0 CUDAX_MACRO_CHECK("B0", termios.h) - -#include <@header@> - -#if defined(CCCL_DISABLE_BF16_SUPPORT) -# if defined(__CUDA_BF16_TYPES_EXIST__) -# error We should not include cuda_bf16.h when BF16 support is disabled -# endif // __CUDA_BF16_TYPES_EXIST__ -#endif // CCCL_DISABLE_BF16_SUPPORT - -#if defined(CCCL_DISABLE_FP16_SUPPORT) -# if defined(__CUDA_FP16_TYPES_EXIST__) -# error We should not include cuda_fp16.h when half support is disabled -# endif // __CUDA_FP16_TYPES_EXIST__ -# if defined(__CUDA_BF16_TYPES_EXIST__) -# error We should not include cuda_bf16.h when half support is disabled -# endif // __CUDA_BF16_TYPES_EXIST__ -#endif // CCCL_DISABLE_FP16_SUPPORT diff --git a/cccl_upstream/cudax/cmake/places_header_unittest.in.cu b/cccl_upstream/cudax/cmake/places_header_unittest.in.cu deleted file mode 100644 index bbf584f6..00000000 --- a/cccl_upstream/cudax/cmake/places_header_unittest.in.cu +++ /dev/null @@ -1,9 +0,0 @@ -// This file is autogenerated by configuring places_header_unittest.in.cu. - -// clang-format off -#define UNITTESTED_FILE "@source@" - -#include - -#include <@source@> -//clang-format on diff --git a/cccl_upstream/cudax/cmake/stf_header_unittest.in.cu b/cccl_upstream/cudax/cmake/stf_header_unittest.in.cu deleted file mode 100644 index 6b2354ce..00000000 --- a/cccl_upstream/cudax/cmake/stf_header_unittest.in.cu +++ /dev/null @@ -1,9 +0,0 @@ -// This file is autogenerated by configuring stf_header_unittest.in.cu. - -// clang-format off -#define UNITTESTED_FILE "@source@" - -#include - -#include <@source@> -//clang-format on diff --git a/cccl_upstream/cudax/examples/CMakeLists.txt b/cccl_upstream/cudax/examples/CMakeLists.txt deleted file mode 100644 index 75fff115..00000000 --- a/cccl_upstream/cudax/examples/CMakeLists.txt +++ /dev/null @@ -1,56 +0,0 @@ -thrust_create_target(cudax.examples.thrust) - -function(cudax_add_example target_name_var example_src) - get_filename_component(example_name ${example_src} NAME_WE) - - # The actual name of the test's target: - set(example_target cudax.example.${example_name}) - set(${target_name_var} ${example_target} PARENT_SCOPE) - - cccl_add_executable(${example_target} SOURCES "${example_src}" ADD_CTEST) - target_link_libraries( - ${example_target} - PRIVATE # - cudax.compiler_interface - cudax.examples.thrust - ) - target_compile_options( - ${example_target} - PRIVATE - $<$:--expt-relaxed-constexpr> - $<$:--extended-lambda> - ) - target_include_directories( - ${example_target} - PRIVATE "${CUB_SOURCE_DIR}/examples" - ) -endfunction() - -file( - GLOB example_srcs - RELATIVE "${cudax_SOURCE_DIR}/examples" - CONFIGURE_DEPENDS - *.cu - *.cpp -) - -cccl_get_cudatoolkit() - -# Example requires pinned_memory_resource. -if (CUDAToolkit_VERSION VERSION_LESS 12.9) - list(REMOVE_ITEM example_srcs async_buffer_add.cu cub_reduce.cu) -endif() - -foreach (example_src IN LISTS example_srcs) - cudax_add_example(example_target "${example_src}") -endforeach() - -# FIXME: Enable MSVC -if (cudax_ENABLE_CUDASTF AND NOT "MSVC" STREQUAL "${CMAKE_CXX_COMPILER_ID}") - # STF examples are handled separately: - add_subdirectory(stf) -endif() - -if (cudax_ENABLE_PLACES AND NOT "MSVC" STREQUAL "${CMAKE_CXX_COMPILER_ID}") - add_subdirectory(places) -endif() diff --git a/cccl_upstream/cudax/examples/async_buffer_add.cu b/cccl_upstream/cudax/examples/async_buffer_add.cu deleted file mode 100644 index 54329f99..00000000 --- a/cccl_upstream/cudax/examples/async_buffer_add.cu +++ /dev/null @@ -1,92 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * Vector addition: C = A + B. - * - * This sample is a very basic sample that implements element by element - * vector addition. It is the same as the sample illustrating Chapter 2 - * of the programming guide with some additions like error checking. - */ - -#include -#include -#include -#include - -#include -#include -#include - -#include - -namespace cudax = cuda::experimental; - -constexpr int numElements = 50000; - -struct generator -{ - thrust::default_random_engine gen{}; - thrust::uniform_real_distribution dist{-10.0f, 10.0f}; - - __host__ __device__ generator(const unsigned seed) - : gen{seed} - {} - - __host__ __device__ float operator()(cuda::std::size_t idx) noexcept - { - gen.discard(idx); - return dist(gen); - } -}; - -int main() -{ - // A CUDA stream on which to execute the vector addition kernel - cudax::stream stream{cuda::device_ref{0}}; - - // The execution policy we want to use to run all work on the same stream - auto policy = thrust::cuda::par_nosync.on(stream.get()); - - cuda::device_memory_pool_ref device_resource = cuda::device_default_memory_pool(cuda::device_ref{0}); - - // Allocate the two inputs and output, but do not zero initialize via `cuda::no_init` - cuda::device_buffer A{stream, device_resource, numElements, cuda::no_init}; - cuda::device_buffer B{stream, device_resource, numElements, cuda::no_init}; - cuda::device_buffer C{stream, device_resource, numElements, cuda::no_init}; - - // Fill both vectors on stream using a random number generator - thrust::tabulate(policy, A.begin(), A.end(), generator{42}); - thrust::tabulate(policy, B.begin(), B.end(), generator{1337}); - - // Add the vectors together - thrust::transform(policy, A.begin(), A.end(), B.begin(), C.begin(), cuda::std::plus<>{}); - - cuda::pinned_memory_pool_ref pinned_resource = cuda::pinned_default_memory_pool(); - - // Verify that the result vector is correct, by copying it to host - cuda::host_buffer h_A{stream, pinned_resource, A}; - cuda::host_buffer h_B{stream, pinned_resource, B}; - cuda::host_buffer h_C{stream, pinned_resource, C}; - - // Do not forget to sync afterwards - stream.sync(); - - for (int i = 0; i < numElements; ++i) - { - if (cuda::std::abs(h_A.get_unsynchronized(i) + h_B.get_unsynchronized(i) - h_C.get_unsynchronized(i)) > 1e-5) - { - std::cerr << "Result verification failed at element " << i << "\n"; - exit(EXIT_FAILURE); - } - } - - return 0; -} diff --git a/cccl_upstream/cudax/examples/cub_reduce.cu b/cccl_upstream/cudax/examples/cub_reduce.cu deleted file mode 100644 index b4614344..00000000 --- a/cccl_upstream/cudax/examples/cub_reduce.cu +++ /dev/null @@ -1,53 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -// Example of using `cub::DeviceReduce::Reduce` with cudax environment. - -#include - -#include -#include -#include - -#include - -namespace cudax = cuda::experimental; - -int main() -{ - constexpr int num_items = 50000; - - // A CUDA stream on which to execute the reduction - cuda::stream stream{cuda::devices[0]}; - cuda::device_memory_pool_ref mr = cuda::device_default_memory_pool(cuda::devices[0]); - - // Allocate input and output, but do not zero initialize output (`cuda::no_init`) - auto d_in = cuda::make_buffer(stream, mr, num_items, 1); - auto d_out = cuda::make_buffer(stream, mr, 1, cuda::no_init); - - // An environment we use to pass all necessary information to CUB - cudax::env_t env{mr, stream}; - auto error = cub::DeviceReduce::Reduce(d_in.begin(), d_out.begin(), num_items, cuda::std::plus{}, 0, env); - if (error != cudaSuccess) - { - std::cerr << "cub::DeviceReduce::Reduce failed: " << cudaGetErrorString(error) << "\n"; - exit(EXIT_FAILURE); - } - - auto h_out = cuda::make_buffer(stream, cuda::pinned_default_memory_pool(), d_out); - - stream.sync(); - - if (h_out.get_unsynchronized(0) != num_items) - { - std::cerr << "Result verification failed: " << h_out.get_unsynchronized(0) << " != " << num_items << "\n"; - exit(EXIT_FAILURE); - } -} diff --git a/cccl_upstream/cudax/examples/places/CMakeLists.txt b/cccl_upstream/cudax/examples/places/CMakeLists.txt deleted file mode 100644 index dc492802..00000000 --- a/cccl_upstream/cudax/examples/places/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -set(places_example_sources thrust_device_data_place_allocator.cu) - -## cudax_add_places_example -# -# Add a places example executable and register it with ctest. -# -# target_name_var: Variable name to overwrite with the name of the example -# target. Useful for modifying the example/target after creation. -# source: The source file for the example. -# -function(cudax_add_places_example target_name_var source) - get_filename_component(filename ${source} NAME_WE) - - set(example_target cudax.example.places.${filename}) - - cccl_add_executable(${example_target} SOURCES ${source} ADD_CTEST) - cudax_places_configure_target(${example_target}) - target_link_libraries( - ${example_target} - PRIVATE # - cudax.compiler_interface - cudax.examples.thrust - ) - - set(${target_name_var} ${example_target} PARENT_SCOPE) -endfunction() - -foreach (source IN LISTS places_example_sources) - cudax_add_places_example(example_target "${source}") -endforeach() diff --git a/cccl_upstream/cudax/examples/places/thrust_device_data_place_allocator.cu b/cccl_upstream/cudax/examples/places/thrust_device_data_place_allocator.cu deleted file mode 100644 index ab88b58f..00000000 --- a/cccl_upstream/cudax/examples/places/thrust_device_data_place_allocator.cu +++ /dev/null @@ -1,117 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Thrust device_vector with an allocator backed by a data_place. - * - * Wraps data_place::allocate/deallocate as a thrust::mr::memory_resource, - * then uses thrust::mr::allocator to create a compatible allocator. - * Storage is allocated via data_place (device, composite/VMM, or other - * place types). The same Thrust code works unchanged for single-device, - * multi-device (VMM), or green-context placement. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -using namespace cuda::experimental::places; - -// Minimal adapter: data_place is STF's abstraction; Thrust expects a -// memory_resource. This class bridges the two. The resource must outlive -// any vectors/allocators that use it. -class data_place_memory_resource final : public thrust::mr::memory_resource> -{ -public: - explicit data_place_memory_resource(const data_place& place) - : place_(place) - {} - - pointer do_allocate(std::size_t bytes, std::size_t /*alignment*/) override - { - // A memory resource hands out untyped bytes, so declare the geometry - // explicitly as a flat byte array: composite places distribute it with - // byte granularity (equivalent for every other place type). - void* raw = place_.allocate_nd(dim4(bytes), 1); - return thrust::device_ptr(raw); - } - - void do_deallocate(pointer p, std::size_t bytes, std::size_t /*alignment*/) override - { - place_.deallocate(p.get(), bytes); - } - - __host__ __device__ bool do_is_equal(const memory_resource& other) const noexcept override - { -#if defined(__CUDA_ARCH__) - (void) other; - return false; -#else - auto* o = dynamic_cast(&other); - return o && place_ == o->place_; -#endif - } - -private: - data_place place_; -}; - -template -using data_place_allocator = thrust::mr::allocator; - -bool run_with_place(const data_place& place, const char* label) -{ - const size_t n = 1024 * 1024; - - data_place_memory_resource memres(place); - data_place_allocator alloc(&memres); - thrust::device_vector> d_vec(n, 0.0, alloc); - - thrust::transform( - thrust::device, - thrust::counting_iterator(0), - thrust::counting_iterator(n), - d_vec.begin(), - [] __device__(size_t i) { - return 2.0 * static_cast(i); - }); - - thrust::host_vector h_sample(4); - thrust::copy(d_vec.begin(), d_vec.begin() + 4, h_sample.begin()); - - bool ok = (h_sample[0] == 0.0 && h_sample[1] == 2.0 && h_sample[2] == 4.0 && h_sample[3] == 6.0); - printf( - "thrust_device_data_place_allocator: %s (%s): %s\n", label, place.to_string().c_str(), ok ? "PASSED" : "FAILED"); - return ok; -} - -int main() -{ - bool all_ok = true; - - all_ok &= run_with_place(data_place::device(0), "device(0)"); - - all_ok &= run_with_place(data_place::composite(blocked_partition(), exec_place::all_devices()), - "composite(blocked_partition, all_devices)"); - - return all_ok ? 0 : 1; -} diff --git a/cccl_upstream/cudax/examples/simple_p2p.cu b/cccl_upstream/cudax/examples/simple_p2p.cu deleted file mode 100644 index 5c021191..00000000 --- a/cccl_upstream/cudax/examples/simple_p2p.cu +++ /dev/null @@ -1,262 +0,0 @@ -/* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of NVIDIA CORPORATION nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY - * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* - * This sample demonstrates a combination of Peer-to-Peer (P2P) and - * Unified Virtual Address Space (UVA) features. - */ - -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -namespace cudax = cuda::experimental; - -struct simple_kernel -{ - template - __device__ void operator()(Configuration config, ::cuda::std::span src, ::cuda::std::span dst) - { - // Just a dummy kernel, doing enough for us to verify that everything worked - const auto idx = cuda::gpu_thread.rank(cuda::grid, config); - dst[idx] = src[idx] * 2.0f; - } -}; - -void print_peer_accessibility() -{ - // Check possibility for peer access - printf("\nChecking GPU(s) for support of peer to peer memory access...\n"); - - for (auto& dev_i : cuda::devices) - { - for (auto& dev_j : cuda::devices) - { - if (dev_i != dev_j) - { - bool can_access_peer = dev_i.has_peer_access_to(dev_j); - const auto dev_i_name = dev_i.name(); - const auto dev_j_name = dev_j.name(); - printf("> Peer access from %.*s (GPU%d) -> %.*s (GPU%d) : %s\n", - static_cast(dev_i_name.size()), - dev_i_name.data(), - dev_i.get(), - static_cast(dev_j_name.size()), - dev_j_name.data(), - dev_j.get(), - can_access_peer ? "Yes" : "No"); - } - } - } -} - -template -void benchmark_cross_device_ping_pong_copy( - cudax::stream_ref dev0_stream, cudax::stream_ref dev1_stream, BufferType& dev0_buffer, BufferType& dev1_buffer) -{ - // Use dev1 stream due to some surprising performance issue - constexpr int cpy_count = 100; - auto start_event = dev1_stream.record_timed_event(); - for (int i = 0; i < cpy_count; i++) - { - // Ping-pong copy between GPUs - if (i % 2 == 0) - { - cuda::copy_bytes(dev1_stream, dev0_buffer, dev1_buffer); - } - else - { - cuda::copy_bytes(dev1_stream, dev1_buffer, dev0_buffer); - } - } - - auto end_event = dev1_stream.record_timed_event(); - dev1_stream.sync(); - cuda::std::chrono::duration duration(end_event - start_event); - printf("Peer copy between GPU%d and GPU%d: %.2fGB/s\n", - dev0_stream.device().get(), - dev1_stream.device().get(), - (static_cast(cpy_count * dev0_buffer.size_bytes()) / static_cast(1024 * 1024 * 1024) - / duration.count())); -} - -template -void test_cross_device_access_from_kernel( - cudax::stream_ref dev0_stream, cudax::stream_ref dev1_stream, BufferType& dev0_buffer, BufferType& dev1_buffer) -{ - cuda::device_ref dev0 = dev0_stream.device(); - cuda::device_ref dev1 = dev1_stream.device(); - - // Prepare host buffer and copy to GPU 0 - printf("Preparing host buffer and copy to GPU%d...\n", dev0.get()); - - // This will be a pinned memory vector once available - cudax::uninitialized_buffer host_buffer( - cuda::mr::legacy_pinned_memory_resource(), dev0_buffer.size()); - std::generate(host_buffer.begin(), host_buffer.end(), []() { - static int i = 0; - return static_cast((i++) % 4096); - }); - - cuda::copy_bytes(dev0_stream, host_buffer, dev0_buffer); - dev1_stream.wait(dev0_stream); - - // Kernel launch configuration - auto config = cuda::distribute<512>(dev0_buffer.size()); - - // Run kernel on GPU 1, reading input from the GPU 0 buffer, writing output to the GPU 1 buffer - printf("Run kernel on GPU%d, taking source data from GPU%d and writing to " - "GPU%d...\n", - dev1.get(), - dev0.get(), - dev1.get()); - cudax::launch(dev1_stream, config, simple_kernel{}, dev0_buffer, dev1_buffer); - dev0_stream.wait(dev1_stream); - - // Run kernel on GPU 0, reading input from the GPU 1 buffer, writing output to the GPU 0 buffer - printf("Run kernel on GPU%d, taking source data from GPU%d and writing to " - "GPU%d...\n", - dev0.get(), - dev1.get(), - dev0.get()); - cudax::launch(dev0_stream, config, simple_kernel{}, dev1_buffer, dev0_buffer); - - // Copy data back to host and verify - printf("Copy data back to host from GPU%d and verify results...\n", dev0.get()); - cuda::copy_bytes(dev0_stream, dev0_buffer, host_buffer); - dev0_stream.sync(); - - int error_count = 0; - for (size_t i = 0; i < host_buffer.size(); i++) - { - cuda::std::span host_span(host_buffer); - // Re-generate input data and apply 2x '* 2.0f' computation of both kernel runs - float expected = float(i % 4096) * 2.0f * 2.0f; - if (host_span[i] != expected) - { - printf("Verification error @ element %zu: val = %f, ref = %f\n", i, host_span[i], expected); - - if (error_count++ > 10) - { - break; - } - } - } - if (error_count != 0) - { - printf("Test failed!\n"); - exit(EXIT_FAILURE); - } -} - -int main([[maybe_unused]] int argc, char** argv) -try -{ - printf("[%s] - Starting...\n", argv[0]); - - // Number of GPUs - printf("Checking for multiple GPUs...\n"); - printf("CUDA-capable device count: %zu\n", cuda::devices.size()); - - if (cuda::devices.size() < 2) - { - printf("Two or more GPUs with Peer-to-Peer access capability are required for %s.\n", argv[0]); - printf("Waiving test.\n"); - return 0; - } - - // Print full peer access matrix - print_peer_accessibility(); - - // But use a shorthand to find all peers of a device - std::vector peers; - for (auto& dev : cuda::devices) - { - const auto dev_peers = dev.peers(); - if (dev_peers.size() != 0) - { - peers.assign(dev_peers.begin(), dev_peers.end()); - peers.insert(peers.begin(), dev); - break; - } - } - - if (peers.size() == 0) - { - printf("Two or more GPUs with Peer-to-Peer access capability are required, waving the test.\n"); - return 0; - } - - cuda::stream dev0_stream(peers[0]); - cuda::stream dev1_stream(peers[1]); - - printf("Enabling peer access between GPU%d and GPU%d...\n", peers[0].get(), peers[1].get()); - cuda::device_memory_pool_ref dev0_resource = cuda::device_default_memory_pool(peers[0]); - dev0_resource.enable_access_from(peers[1]); - cuda::device_memory_pool_ref dev1_resource = cuda::device_default_memory_pool(peers[1]); - dev1_resource.enable_access_from(peers[0]); - - // Allocate buffers - constexpr size_t buf_cnt = 1024 * 1024 * 16; - printf("Allocating buffers (%iMB on GPU%d, GPU%d and CPU Host)...\n", - int(buf_cnt / 1024 / 1024 * sizeof(float)), - peers[0].get(), - peers[1].get()); - - cudax::uninitialized_buffer dev0_buffer(dev0_resource, buf_cnt); - cudax::uninitialized_buffer dev1_buffer(dev1_resource, buf_cnt); - - benchmark_cross_device_ping_pong_copy(dev0_stream, dev1_stream, dev0_buffer, dev1_buffer); - - test_cross_device_access_from_kernel(dev0_stream, dev1_stream, dev0_buffer, dev1_buffer); - - // Disable peer access - printf("Disabling peer access...\n"); - dev0_resource.disable_access_from(peers[1]); - dev1_resource.disable_access_from(peers[0]); - - // No cleanup needed - printf("Test passed\n"); - return 0; -} -catch (const std::exception& e) -{ - printf("caught an exception: \"%s\"\n", e.what()); -} -catch (...) -{ - printf("caught an unknown exception\n"); -} diff --git a/cccl_upstream/cudax/examples/stdexec_stream.cu b/cccl_upstream/cudax/examples/stdexec_stream.cu deleted file mode 100644 index b0760487..00000000 --- a/cccl_upstream/cudax/examples/stdexec_stream.cu +++ /dev/null @@ -1,82 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#include - -#include - -#include - -#include - -namespace cudax = cuda::experimental; -namespace ex = cudax::execution; - -// This example demonstrates how to use the experimental CUDA implementation of -// C++26's std::execution async tasking framework. - -int main() -{ - try - { - auto tctx = ex::thread_context{}; - auto sctx = ex::stream_context{cuda::device_ref{0}}; - auto gpu = sctx.get_scheduler(); - - const auto bulk_shape = 10; - const auto bulk_fn = [] __device__(const int index, int i) noexcept { - const int tid = static_cast(blockIdx.x * blockDim.x + threadIdx.x); - if (tid < bulk_shape) - { - printf("Hello from bulk task on device! index = %d, i = %d\n", index, i); - } - }; - - auto start = - // begin work on the GPU: - ex::schedule(gpu) - - // execute a device lambda on the GPU: - | ex::then([] __device__() noexcept -> int { - printf("Hello from lambda on device!\n"); - return 42; - }) - - // do some parallel work on the GPU: - | ex::bulk(ex::par, bulk_shape, bulk_fn) // - - // transfer execution back to the CPU: - | ex::continues_on(tctx.get_scheduler()) - - // execute a host/device lambda on the CPU: - | ex::then([] __host__ __device__(int i) noexcept -> int { - NV_IF_ELSE_TARGET(NV_IS_HOST, - (printf("Hello from lambda on host! i = %d\n", i);), - (printf("OOPS! still on the device! i = %d\n", i);)) - return i + 1; - }); - - // run the task, wait for it to finish, and get the result - auto [i] = ex::sync_wait(std::move(start)).value(); - printf("All done on the host! result = %d\n", i); - } - catch (cuda::cuda_error const& e) - { - std::printf("CUDA error: %s\n", e.what()); - } - catch (std::exception const& e) - { - std::printf("Exception: %s\n", e.what()); - } - catch (...) - { - std::printf("Unknown exception\n"); - } -} diff --git a/cccl_upstream/cudax/examples/stf/01-axpy-cuda_kernel.cu b/cccl_upstream/cudax/examples/stf/01-axpy-cuda_kernel.cu deleted file mode 100644 index 4613bac5..00000000 --- a/cccl_upstream/cudax/examples/stf/01-axpy-cuda_kernel.cu +++ /dev/null @@ -1,73 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief An AXPY kernel described using a cuda_kernel construct - * - */ - -#include - -using namespace cuda::experimental::stf; - -__global__ void axpy(double a, slice x, slice y) -{ - int tid = blockIdx.x * blockDim.x + threadIdx.x; - int nthreads = gridDim.x * blockDim.x; - - for (int i = tid; i < x.size(); i += nthreads) - { - y(i) += a * x(i); - } -} - -double X0(int i) -{ - return sin((double) i); -} - -double Y0(int i) -{ - return cos((double) i); -} - -int main() -{ - context ctx = graph_ctx(); - const size_t N = 16; - double X[N], Y[N]; - - for (size_t i = 0; i < N; i++) - { - X[i] = X0(i); - Y[i] = Y0(i); - } - - double alpha = 3.14; - - auto lX = ctx.logical_data(X); - auto lY = ctx.logical_data(Y); - - /* Compute Y = Y + alpha X */ - ctx.cuda_kernel(lX.read(), lY.rw())->*[&](auto dX, auto dY) { - // axpy<<<16, 128, 0, ...>>>(alpha, dX, dY) - return cuda_kernel_desc{axpy, 16, 128, 0, alpha, dX, dY}; - }; - - ctx.finalize(); - - for (size_t i = 0; i < N; i++) - { - assert(fabs(Y[i] - (Y0(i) + alpha * X0(i))) < 0.0001); - assert(fabs(X[i] - X0(i)) < 0.0001); - } -} diff --git a/cccl_upstream/cudax/examples/stf/01-axpy-cuda_kernel_chain.cu b/cccl_upstream/cudax/examples/stf/01-axpy-cuda_kernel_chain.cu deleted file mode 100644 index fb3724dc..00000000 --- a/cccl_upstream/cudax/examples/stf/01-axpy-cuda_kernel_chain.cu +++ /dev/null @@ -1,80 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Example of task implementing a chain of CUDA kernels - * - */ - -#include - -using namespace cuda::experimental::stf; - -__global__ void axpy(double a, slice x, slice y) -{ - int tid = blockIdx.x * blockDim.x + threadIdx.x; - int nthreads = gridDim.x * blockDim.x; - - for (int i = tid; i < x.size(); i += nthreads) - { - y(i) += a * x(i); - } -} - -double X0(int i) -{ - return sin((double) i); -} - -double Y0(int i) -{ - return cos((double) i); -} - -int main() -{ - context ctx = graph_ctx(); - const size_t N = 16; - double X[N], Y[N]; - - for (size_t i = 0; i < N; i++) - { - X[i] = X0(i); - Y[i] = Y0(i); - } - - double alpha = 3.14; - double beta = 4.5; - double gamma = -4.1; - - auto lX = ctx.logical_data(X); - auto lY = ctx.logical_data(Y); - - /* Compute Y = Y + alpha X, Y = Y + beta X and then Y = Y + gamma X */ - ctx.cuda_kernel_chain(lX.read(), lY.rw())->*[&](auto dX, auto dY) { - // clang-format off - return std::vector { - { axpy, 16, 128, 0, alpha, dX, dY }, - { axpy, 16, 128, 0, beta, dX, dY }, - { axpy, 16, 128, 0, gamma, dX, dY } - }; - // clang-format on - }; - - ctx.finalize(); - - for (size_t i = 0; i < N; i++) - { - assert(fabs(Y[i] - (Y0(i) + (alpha + beta + gamma) * X0(i))) < 0.0001); - assert(fabs(X[i] - X0(i)) < 0.0001); - } -} diff --git a/cccl_upstream/cudax/examples/stf/01-axpy-launch.cu b/cccl_upstream/cudax/examples/stf/01-axpy-launch.cu deleted file mode 100644 index 4dcd01ab..00000000 --- a/cccl_upstream/cudax/examples/stf/01-axpy-launch.cu +++ /dev/null @@ -1,62 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Example of AXPY kernel implemented with the launch API - */ - -#include - -using namespace cuda::experimental::stf; - -double X0(int i) -{ - return sin((double) i); -} - -double Y0(int i) -{ - return cos((double) i); -} - -int main() -{ - context ctx; - const size_t N = 16; - double X[N], Y[N]; - - for (size_t i = 0; i < N; i++) - { - X[i] = X0(i); - Y[i] = Y0(i); - } - - double alpha = 3.14; - - auto lX = ctx.logical_data(X); - auto lY = ctx.logical_data(Y); - - /* Compute Y = Y + alpha X */ - ctx.launch(lX.read(), lY.rw())->*[=] _CCCL_DEVICE(auto t, auto dX, auto dY) { - for (auto ind : t.apply_partition(shape(dX))) - { - dY(ind) += alpha * dX(ind); - } - }; - - ctx.finalize(); - - for (size_t i = 0; i < N; i++) - { - assert(fabs(Y[i] - (Y0(i) + alpha * X0(i))) < 0.0001); - assert(fabs(X[i] - X0(i)) < 0.0001); - } -} diff --git a/cccl_upstream/cudax/examples/stf/01-axpy-parallel_for.cu b/cccl_upstream/cudax/examples/stf/01-axpy-parallel_for.cu deleted file mode 100644 index be69b8db..00000000 --- a/cccl_upstream/cudax/examples/stf/01-axpy-parallel_for.cu +++ /dev/null @@ -1,61 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief An AXPY kernel implemented using the parallel_for construct - * - */ - -#include - -using namespace cuda::experimental::stf; - -double X0(int i) -{ - return sin((double) i); -} - -double Y0(int i) -{ - return cos((double) i); -} - -int main() -{ - context ctx; - const size_t N = 16; - double X[N], Y[N]; - - for (size_t i = 0; i < N; i++) - { - X[i] = X0(i); - Y[i] = Y0(i); - } - - double alpha = 3.14; - - auto lX = ctx.logical_data(X); - auto lY = ctx.logical_data(Y); - - /* Compute Y = Y + alpha X */ - ctx.parallel_for(lY.shape(), lX.read(), lY.rw())->*[alpha] __device__(size_t i, auto dX, auto dY) { - dY(i) += alpha * dX(i); - }; - - ctx.finalize(); - - for (size_t i = 0; i < N; i++) - { - assert(fabs(Y[i] - (Y0(i) + alpha * X0(i))) < 0.0001); - assert(fabs(X[i] - X0(i)) < 0.0001); - } -} diff --git a/cccl_upstream/cudax/examples/stf/01-axpy.cu b/cccl_upstream/cudax/examples/stf/01-axpy.cu deleted file mode 100644 index 5c1d684f..00000000 --- a/cccl_upstream/cudax/examples/stf/01-axpy.cu +++ /dev/null @@ -1,72 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief An AXPY kernel implemented with CUDA kernel in a task - * - */ - -#include - -using namespace cuda::experimental::stf; - -__global__ void axpy(double a, slice x, slice y) -{ - int tid = blockIdx.x * blockDim.x + threadIdx.x; - int nthreads = gridDim.x * blockDim.x; - - for (int i = tid; i < x.size(); i += nthreads) - { - y(i) += a * x(i); - } -} - -double X0(int i) -{ - return sin((double) i); -} - -double Y0(int i) -{ - return cos((double) i); -} - -int main() -{ - context ctx; - const size_t N = 16; - double X[N], Y[N]; - - for (size_t i = 0; i < N; i++) - { - X[i] = X0(i); - Y[i] = Y0(i); - } - - double alpha = 3.14; - - auto lX = ctx.logical_data(X); - auto lY = ctx.logical_data(Y); - - /* Compute Y = Y + alpha X */ - ctx.task(lX.read(), lY.rw())->*[&](cudaStream_t s, auto dX, auto dY) { - axpy<<<16, 128, 0, s>>>(alpha, dX, dY); - }; - - ctx.finalize(); - - for (size_t i = 0; i < N; i++) - { - assert(fabs(Y[i] - (Y0(i) + alpha * X0(i))) < 0.0001); - assert(fabs(X[i] - X0(i)) < 0.0001); - } -} diff --git a/cccl_upstream/cudax/examples/stf/02-axpy-host_launch.cu b/cccl_upstream/cudax/examples/stf/02-axpy-host_launch.cu deleted file mode 100644 index 5c7bb85b..00000000 --- a/cccl_upstream/cudax/examples/stf/02-axpy-host_launch.cu +++ /dev/null @@ -1,81 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief An AXPY kernel implemented with a task of the CUDA graph backend and - * a host callback - * - * The host_launch mechanism is also illustrated - * - */ - -#include - -using namespace cuda::experimental::stf; - -__global__ void axpy(double a, slice x, slice y) -{ - int tid = blockIdx.x * blockDim.x + threadIdx.x; - int nthreads = gridDim.x * blockDim.x; - - for (int i = tid; i < x.size(); i += nthreads) - { - y(i) += a * x(i); - } -} - -double X0(int i) -{ - return sin((double) i); -} - -double Y0(int i) -{ - return cos((double) i); -} - -int main() -{ - graph_ctx ctx; - const size_t N = 16; - double X[N], Y[N]; - - for (size_t i = 0; i < N; i++) - { - X[i] = X0(i); - Y[i] = Y0(i); - } - - double alpha = 3.14; - - auto lX = ctx.logical_data(X); - auto lY = ctx.logical_data(Y); - - /* Compute Y = Y + alpha X */ - ctx.task(lX.read(), lY.rw())->*[&](cudaStream_t s, auto dX, auto dY) { - axpy<<<16, 128, 0, s>>>(alpha, dX, dY); - }; - - /* Asynchronously check the result on the host */ - ctx.host_launch(lX.read(), lY.read())->*[&](auto hX, auto hY) { - for (size_t ind = 0; ind < hX.extent(0); ind++) - { - // Y should be Y0 + alpha X0 - EXPECT(fabs(hY(ind) - (Y0(ind) + alpha * X0(ind))) < 0.0001); - - // X should be X0 - EXPECT(fabs(hX(ind) - X0(ind)) < 0.0001); - } - }; - - ctx.finalize(); -} diff --git a/cccl_upstream/cudax/examples/stf/03-temporary-data.cu b/cccl_upstream/cudax/examples/stf/03-temporary-data.cu deleted file mode 100644 index 491cdd57..00000000 --- a/cccl_upstream/cudax/examples/stf/03-temporary-data.cu +++ /dev/null @@ -1,69 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief This example illustrates how we can create temporary data from shapes, and use them in tasks - */ - -#include - -using namespace cuda::experimental::stf; - -int main() -{ - const int n = 4096; - int X[n]; - int Y[n]; - - for (size_t i = 0; i < n; i++) - { - X[i] = 3 * i; - Y[i] = 2 * i - 3; - } - - context ctx; - - auto lX = ctx.logical_data(X); - auto lY = ctx.logical_data(Y); - - // Select an odd number - int niter = 19; - assert(niter % 2 == 1); - - for (int iter = 0; iter < niter; iter++) - { - // We here define a temporary vector with the same shape as X, for which there is no existing copy - // This data handle has a limited scope, so that it is automatically destroyed at each iteration of the loop - auto tmp = ctx.logical_data(lX.shape()); - - ctx.task(lY.rw(), lX.rw(), tmp.write())->*[](cudaStream_t s, auto sY, auto sX, auto sTMP) { - // We swap X and Y using TMP as temporary buffer - // TMP = X - cuda_safe_call( - cudaMemcpyAsync(sTMP.data_handle(), sX.data_handle(), n * sizeof(int), cudaMemcpyDeviceToDevice, s)); - // X = Y - cuda_safe_call(cudaMemcpyAsync(sX.data_handle(), sY.data_handle(), n * sizeof(int), cudaMemcpyDeviceToDevice, s)); - // Y = TMP - cuda_safe_call( - cudaMemcpyAsync(sY.data_handle(), sTMP.data_handle(), n * sizeof(int), cudaMemcpyDeviceToDevice, s)); - }; - } - - ctx.finalize(); - - // We have exchanged an odd number of times, so they must be inverted - for (size_t i = 0; i < n; i++) - { - assert(X[i] == 2 * i - 3); - assert(Y[i] == 3 * i); - } -} diff --git a/cccl_upstream/cudax/examples/stf/04-fibonacci-run_once.cu b/cccl_upstream/cudax/examples/stf/04-fibonacci-run_once.cu deleted file mode 100644 index 61b6cc36..00000000 --- a/cccl_upstream/cudax/examples/stf/04-fibonacci-run_once.cu +++ /dev/null @@ -1,79 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief An example of Fibonacci sequence illustrating how we can use - * dynamically created logical data and the run_once utility - */ - -#include - -using namespace cuda::experimental::stf; - -int fibo_ref(int n) -{ - if (n < 2) - { - return n; - } - else - { - return fibo_ref(n - 1) + fibo_ref(n - 2); - } -} - -__global__ void add(slice out, const slice in1, const slice in2) -{ - out(0) = in1(0) + in2(0); -} - -__global__ void set(slice out, int val) -{ - out(0) = val; -} - -logical_data> compute_fibo(context& ctx, int n) -{ - // The result for a given value n is memoized in a logical_data that will be reused every time we compute the same - // value - return run_once(n)->*[&](int n) { - auto result = ctx.logical_data(shape_of>(1)).set_symbol(std::to_string(n)); - if (n < 2) - { - ctx.task(result.write()).set_symbol("fibo" + std::to_string(n))->*[=](cudaStream_t s, auto sresult) { - set<<<1, 1, 0, s>>>(sresult, n); - }; - } - else - { - auto fib2 = compute_fibo(ctx, n - 2); - auto fib1 = compute_fibo(ctx, n - 1); - ctx.task(fib1.read(), fib2.read(), result.write()).set_symbol("fibo" + std::to_string(n)) - ->*[=](cudaStream_t s, auto s1, auto s2, auto sresult) { - add<<<1, 1, 0, s>>>(sresult, s1, s2); - }; - } - return result; - }; -} - -int main(int argc, char** argv) -{ - int n = (argc > 1) ? atoi(argv[1]) : 4; - - context ctx; - auto result = compute_fibo(ctx, n); - ctx.host_launch(result.read())->*[&](auto res) { - EXPECT(res(0) == fibo_ref(n)); - }; - ctx.finalize(); -} diff --git a/cccl_upstream/cudax/examples/stf/04-fibonacci.cu b/cccl_upstream/cudax/examples/stf/04-fibonacci.cu deleted file mode 100644 index c42a3842..00000000 --- a/cccl_upstream/cudax/examples/stf/04-fibonacci.cu +++ /dev/null @@ -1,75 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief An example of Fibonacci sequence illustrating how we can use - * dynamically created logical data - */ - -#include - -using namespace cuda::experimental::stf; - -int fibo_ref(int n) -{ - if (n < 2) - { - return n; - } - else - { - return fibo_ref(n - 1) + fibo_ref(n - 2); - } -} - -__global__ void add(slice out, const slice in1, const slice in2) -{ - out(0) = in1(0) + in2(0); -} - -__global__ void set(slice out, int val) -{ - out(0) = val; -} - -logical_data> compute_fibo(context& ctx, int n) -{ - auto out = ctx.logical_data(shape_of>(1)); - if (n < 2) - { - ctx.task(out.write())->*[=](cudaStream_t s, auto sout) { - set<<<1, 1, 0, s>>>(sout, n); - }; - } - else - { - auto fib1 = compute_fibo(ctx, n - 1); - auto fib2 = compute_fibo(ctx, n - 2); - ctx.task(fib1.read(), fib2.read(), out.write())->*[=](cudaStream_t s, auto s1, auto s2, auto sout) { - add<<<1, 1, 0, s>>>(sout, s1, s2); - }; - } - - return out; -} - -int main(int argc, char** argv) -{ - int n = (argc > 1) ? atoi(argv[1]) : 4; - - context ctx; // = graph_ctx(); - auto result = compute_fibo(ctx, n); - ctx.host_launch(result.read())->*[&](auto res) { - EXPECT(res(0) == fibo_ref(n)); - }; - ctx.finalize(); -} diff --git a/cccl_upstream/cudax/examples/stf/08-cub-reduce.cu b/cccl_upstream/cudax/examples/stf/08-cub-reduce.cu deleted file mode 100644 index 0695c90d..00000000 --- a/cccl_upstream/cudax/examples/stf/08-cub-reduce.cu +++ /dev/null @@ -1,95 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Example of reduction implementing using CUB kernels - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -template -__global__ void reduce(slice values, slice partials, size_t nelems) -{ - using namespace cub; - typedef BlockReduce BlockReduceT; - - auto thread_id = BLOCK_THREADS * blockIdx.x + threadIdx.x; - - // Local reduction - T local_sum = 0; - for (size_t ind = thread_id; ind < nelems; ind += blockDim.x * gridDim.x) - { - local_sum += values(ind); - } - - __shared__ typename BlockReduceT::TempStorage temp_storage; - - // Per-thread tile data - T result = BlockReduceT(temp_storage).Sum(local_sum); - - if (threadIdx.x == 0) - { - partials(blockIdx.x) = result; - } -} - -template -void run() -{ - Ctx ctx; - - const size_t N = 1024 * 16; - const size_t BLOCK_SIZE = 128; - const size_t num_blocks = 32; - - int *X, ref_tot; - - X = new int[N]; - ref_tot = 0; - - for (size_t ind = 0; ind < N; ind++) - { - X[ind] = rand() % N; - ref_tot += X[ind]; - } - - auto values = ctx.logical_data(X, {N}); - auto partials = ctx.logical_data(shape_of>(num_blocks)); - auto result = ctx.logical_data(shape_of>(1)); - - ctx.task(values.read(), partials.write(), result.write())->*[&](auto stream, auto values, auto partials, auto result) { - // reduce values into partials - reduce<<>>(values, partials, N); - - // reduce partials on a single block into result - reduce<<<1, BLOCK_SIZE, 0, stream>>>(partials, result, num_blocks); - }; - - ctx.host_launch(result.read())->*[&](auto p) { - if (p(0) != ref_tot) - { - fprintf(stderr, "INCORRECT RESULT: p sum = %d, ref tot = %d\n", p(0), ref_tot); - abort(); - } - }; - - ctx.finalize(); -} - -int main() -{ - run(); - run(); -} diff --git a/cccl_upstream/cudax/examples/stf/09-dot-reduce.cu b/cccl_upstream/cudax/examples/stf/09-dot-reduce.cu deleted file mode 100644 index a3c5f60f..00000000 --- a/cccl_upstream/cudax/examples/stf/09-dot-reduce.cu +++ /dev/null @@ -1,55 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Implementation of the DOT kernel using a reduce access mode - * - */ - -#include - -using namespace cuda::experimental::stf; - -int main() -{ - const size_t N = 16; - double X[N], Y[N]; - - double ref_res = 0.0; - - for (size_t i = 0; i < N; i++) - { - X[i] = cos(double(i)); - Y[i] = sin(double(i)); - - // Compute the reference result of the DOT product of X and Y - ref_res += X[i] * Y[i]; - } - - context ctx; - auto lX = ctx.logical_data(X); - auto lY = ctx.logical_data(Y); - - auto lsum = ctx.logical_data(shape_of>()); - - /* Compute sum(x_i * y_i)*/ - ctx.parallel_for(lY.shape(), lX.read(), lY.read(), lsum.reduce(reducer::sum{})) - ->*[] __device__(size_t i, auto dX, auto dY, double& sum) { - sum += dX(i) * dY(i); - }; - - double res = ctx.wait(lsum); - - ctx.finalize(); - - _CCCL_ASSERT(fabs(res - ref_res) < 0.0001, "Invalid result"); -} diff --git a/cccl_upstream/cudax/examples/stf/1f1b.cu b/cccl_upstream/cudax/examples/stf/1f1b.cu deleted file mode 100644 index ac032077..00000000 --- a/cccl_upstream/cudax/examples/stf/1f1b.cu +++ /dev/null @@ -1,118 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Toy example to reproduce the asynchrony of a 1F1B pipeline - */ - -#include - -using namespace cuda::experimental::stf; - -__global__ void forward(slice, long long int clock_cnt) -{ - long long int start_clock = clock64(); - long long int clock_offset = 0; - while (clock_offset < clock_cnt) - { - clock_offset = clock64() - start_clock; - } -} - -__global__ void backward(slice, long long int clock_cnt) -{ - long long int start_clock = clock64(); - long long int clock_offset = 0; - while (clock_offset < clock_cnt) - { - clock_offset = clock64() - start_clock; - } -} - -int main(int argc, char** argv) -{ - context ctx; - // Use a graph context if the second argument is set and not null - if (argc > 2 && atoi(argv[2])) - { - ctx = graph_ctx(); - } - - int device; - cudaGetDevice(&device); - - // cudaDevAttrClockRate: Peak clock frequency in kilohertz; - int clock_rate; - cudaDeviceGetAttribute(&clock_rate, cudaDevAttrClockRate, device); - - auto occ_f = reserved::compute_occupancy(forward); - auto occ_b = reserved::compute_occupancy(backward); - - int factor = 1; - if (argc > 1) - { - factor = atoi(argv[1]); - } - - size_t num_batches = 8 * factor; - int num_devs = 8; - int real_devs; - cuda_safe_call(cudaGetDeviceCount(&real_devs)); - - std::vector>> data; - - for (size_t b = 0; b < num_batches; b++) - { - auto batch_data = ctx.logical_data(shape_of>(1024)); - data.push_back(batch_data); - - ctx.task(exec_place::device(0), data[b].write())->*[](cudaStream_t, auto) { - // Init ... - }; - } - - cuda_safe_call(cudaStreamSynchronize(ctx.fence())); - - size_t niter = 10; - - for (size_t iter = 0; iter < niter; iter++) - { - for (size_t b = 0; b < num_batches; b++) - { - for (int d = 0; d < num_devs; d++) - { - ctx.task(exec_place::device(d % real_devs), data[b].rw())->*[=](cudaStream_t s, auto bd) { - int ms = 10; - long long int clock_cnt = (long long int) (ms * clock_rate / factor); - forward<<>>(bd, clock_cnt); - }; - } - // } - // - // for (size_t b = 0; b < num_batches; b++) { - for (int d = num_devs; d-- > 0;) - { - ctx.task(exec_place::device(d % real_devs), data[b].rw())->*[=](cudaStream_t s, auto bd) { - int ms = 20; - long long int clock_cnt = (long long int) (ms * clock_rate / factor); - backward<<>>(bd, clock_cnt); - }; - } - } - - /* We introduce a fence because the actual pipeline would introduce - * some all to all communication to update coefficients */ - cuda_safe_call(cudaStreamSynchronize(ctx.fence())); - } - - ctx.finalize(); - return 0; -} diff --git a/cccl_upstream/cudax/examples/stf/CMakeLists.txt b/cccl_upstream/cudax/examples/stf/CMakeLists.txt deleted file mode 100644 index 963eec02..00000000 --- a/cccl_upstream/cudax/examples/stf/CMakeLists.txt +++ /dev/null @@ -1,124 +0,0 @@ -set( - stf_example_sources - 01-axpy.cu - 01-axpy-cuda_kernel.cu - 01-axpy-cuda_kernel_chain.cu - 02-axpy-host_launch.cu - 03-temporary-data.cu - 04-fibonacci.cu - 04-fibonacci-run_once.cu - 08-cub-reduce.cu - axpy-annotated.cu - void_data_interface.cu - explicit_data_places.cu - partitioned_axpy.cu - thrust_zip_iterator.cu - 1f1b.cu -) - -# Examples which rely on code generation (parallel_for or launch) -set( - stf_example_codegen_sources - 01-axpy-launch.cu - 01-axpy-parallel_for.cu - binary_fhe.cu - binary_fhe_stackable.cu - 09-dot-reduce.cu - cfd.cu - custom_data_interface.cu - fdtd_mgpu.cu - fdtd_while.cu - fdtd_repeat_n.cu - frozen_data_init.cu - graph_algorithms/degree_centrality.cu - graph_algorithms/jaccard.cu - graph_algorithms/pagerank.cu - graph_algorithms/pagerank_batched.cu - graph_algorithms/pagerank_while.cu - graph_algorithms/tricount.cu - graph_scope.cu - heat.cu - heat_mgpu.cu - jacobi.cu - jacobi_pfor.cu - jacobi_stackable.cu - jacobi_stackable_raii.cu - jacobi_update_cond.cu - launch_histogram.cu - launch_scan.cu - launch_sum.cu - launch_sum_cub.cu - linear_algebra/burger.cu - linear_algebra/burger_sensitivity.cu - linear_algebra/cg_csr.cu - linear_algebra/cg_csr_stackable.cu - logical_gates_composition.cu - mandelbrot.cu - parallel_for_2D.cu - pi.cu - scan.cu - sqrt_newton_stackable.cu - standalone-launches.cu - word_count.cu - word_count_reduce.cu -) - -# Examples using CUBLAS, CUSOLVER... -set( - stf_example_mathlib_sources - linear_algebra/06-pdgemm.cu - linear_algebra/06-pdgemm-stackable.cu - linear_algebra/07-cholesky.cu - linear_algebra/07-potri.cu - linear_algebra/cg_dense_2D.cu - linear_algebra/strassen.cu -) - -cccl_get_cudatoolkit() - -## cudax_add_stf_example -# -# Add an stf example executable and register it with ctest. -# -# target_name_var: Variable name to overwrite with the name of the example -# target. Useful for modifying the example/target after creation. -# source: The source file for the example. -# -# Additional args are passed to cudax_stf_configure_target. -function(cudax_add_stf_example target_name_var source) - get_filename_component(dir ${source} DIRECTORY) - get_filename_component(filename ${source} NAME_WE) - if (dir) - set(filename "${dir}/${filename}") - endif() - string(REPLACE "/" "." example_name "stf/${filename}") - - set(example_target cudax.example.${example_name}) - - cccl_add_executable(${example_target} SOURCES ${source} ADD_CTEST) - cudax_stf_configure_target(${example_target} ${ARGN}) - target_link_libraries( - ${example_target} - PRIVATE # - cudax.compiler_interface - cudax.examples.thrust - ) - - set(${target_name_var} ${example_target} PARENT_SCOPE) -endfunction() - -foreach (source IN LISTS stf_example_sources) - cudax_add_stf_example(example_target "${source}") -endforeach() - -if (cudax_ENABLE_CUDASTF_CODE_GENERATION) - foreach (source IN LISTS stf_example_codegen_sources) - cudax_add_stf_example(example_target "${source}") - endforeach() -endif() - -if (cudax_ENABLE_CUDASTF_MATHLIBS) - foreach (source IN LISTS stf_example_mathlib_sources) - cudax_add_stf_example(example_target "${source}" LINK_MATHLIBS) - endforeach() -endif() diff --git a/cccl_upstream/cudax/examples/stf/axpy-annotated.cu b/cccl_upstream/cudax/examples/stf/axpy-annotated.cu deleted file mode 100644 index f4b81f5e..00000000 --- a/cccl_upstream/cudax/examples/stf/axpy-annotated.cu +++ /dev/null @@ -1,82 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief This example illustrates how we can annotate tasks and logical data with debugging symbol - * - * CUDASTF_DOT_FILE=axpy.dot build/examples/axpy-annotated - * - * # Generate the visualization from this dot file in PDF or PNG format - * dot -Tpdf axpy.dot -o axpy.pdf - * dot -Tpng axpy.dot -o axpy.png - * - * # Generate visualization with events (for advanced users) - * CUDASTF_DOT_IGNORE_PREREQS=0 CUDASTF_DOT_FILE=axpy-with-events.dot build/examples/axpy-annotated - * dot -Tpng axpy-with-events.dot -o axpy-with-events.png - * - */ - -#include - -using namespace cuda::experimental::stf; - -__global__ void axpy(double a, slice x, slice y) -{ - int tid = blockIdx.x * blockDim.x + threadIdx.x; - int nthreads = gridDim.x * blockDim.x; - - for (int i = tid; i < x.size(); i += nthreads) - { - y(i) += a * x(i); - } -} - -double X0(int i) -{ - return sin((double) i); -} - -double Y0(int i) -{ - return cos((double) i); -} - -int main() -{ - context ctx; - const size_t N = 16; - double X[N], Y[N]; - - for (size_t i = 0; i < N; i++) - { - X[i] = X0(i); - Y[i] = Y0(i); - } - - double alpha = 3.14; - - auto lX = ctx.logical_data(X).set_symbol("X"); - auto lY = ctx.logical_data(Y).set_symbol("Y"); - - /* Compute Y = Y + alpha X */ - ctx.task(lX.read(), lY.rw()).set_symbol("axpy")->*[&](cudaStream_t s, auto dX, auto dY) { - axpy<<<16, 128, 0, s>>>(alpha, dX, dY); - }; - - ctx.finalize(); - - for (size_t i = 0; i < N; i++) - { - assert(fabs(Y[i] - (Y0(i) + alpha * X0(i))) < 0.0001); - assert(fabs(X[i] - X0(i)) < 0.0001); - } -} diff --git a/cccl_upstream/cudax/examples/stf/binary_fhe.cu b/cccl_upstream/cudax/examples/stf/binary_fhe.cu deleted file mode 100644 index 6b6e8615..00000000 --- a/cccl_upstream/cudax/examples/stf/binary_fhe.cu +++ /dev/null @@ -1,201 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief A toy example to illustrate how we can compose logical operations - * over encrypted data - */ - -#include - -using namespace cuda::experimental::stf; - -class ciphertext; - -class plaintext -{ -public: - plaintext(const context& ctx) - : ctx(ctx) - {} - - plaintext(context& ctx, std::vector v) - : values(v) - , ctx(ctx) - { - l = ctx.logical_data(&values[0], values.size()); - } - - void set_symbol(std::string s) - { - l.set_symbol(s); - symbol = s; - } - - std::string get_symbol() const - { - return symbol; - } - - std::string symbol; - - const logical_data>& data() const - { - return l; - } - - logical_data>& data() - { - return l; - } - - // This will asynchronously fill string s - void convert_to_vector(std::vector& v) - { - ctx.host_launch(l.read()).set_symbol("to_vector")->*[&](auto dl) { - v.resize(dl.size()); - for (size_t i = 0; i < dl.size(); i++) - { - v[i] = dl(i); - } - }; - } - - ciphertext encrypt() const; - - logical_data> l; - -private: - std::vector values; - mutable context ctx; -}; - -class ciphertext -{ -public: - ciphertext(const context& ctx) - : ctx(ctx) - {} - - plaintext decrypt() const - { - plaintext p(ctx); - p.l = ctx.logical_data(shape_of>(l.shape().size())); - // fprintf(stderr, "Decrypting...\n"); - ctx.parallel_for(l.shape(), l.read(), p.l.write()).set_symbol("decrypt")->* - [] _CCCL_DEVICE(size_t i, auto dctxt, auto dptxt) { - dptxt(i) = char((dctxt(i) >> 32)); - // printf("DECRYPT %ld : %lx -> %x\n", i, dctxt(i), (int) dptxt(i)); - }; - return p; - } - - ciphertext operator|(const ciphertext& other) const - { - ciphertext result(ctx); - result.l = ctx.logical_data(data().shape()); - - ctx.parallel_for(data().shape(), data().read(), other.data().read(), result.data().write()).set_symbol("OR")->* - [] _CCCL_DEVICE(size_t i, auto d_c1, auto d_c2, auto d_res) { - d_res(i) = d_c1(i) | d_c2(i); - }; - - return result; - } - - ciphertext operator&(const ciphertext& other) const - { - ciphertext result(ctx); - result.l = ctx.logical_data(data().shape()); - - ctx.parallel_for(data().shape(), data().read(), other.data().read(), result.data().write()).set_symbol("AND")->* - [] _CCCL_DEVICE(size_t i, auto d_c1, auto d_c2, auto d_res) { - d_res(i) = d_c1(i) & d_c2(i); - }; - - return result; - } - - ciphertext operator~() const - { - ciphertext result(ctx); - result.l = ctx.logical_data(data().shape()); - ctx.parallel_for(data().shape(), data().read(), result.data().write()).set_symbol("NOT")->* - [] _CCCL_DEVICE(size_t i, auto d_c, auto d_res) { - d_res(i) = ~d_c(i); - }; - - return result; - } - - const logical_data>& data() const - { - return l; - } - - logical_data>& data() - { - return l; - } - - logical_data> l; - -private: - mutable context ctx; -}; - -ciphertext plaintext::encrypt() const -{ - ciphertext c(ctx); - c.l = ctx.logical_data(shape_of>(l.shape().size())); - - ctx.parallel_for(l.shape(), l.read(), c.l.write()).set_symbol("encrypt")->* - [] _CCCL_DEVICE(size_t i, auto dptxt, auto dctxt) { - // A super safe encryption ! - dctxt(i) = ((uint64_t) (dptxt(i)) << 32 | 0x4); - }; - - return c; -} - -template -T circuit(const T& a, const T& b) -{ - return (~((a | ~b) & (~a | b))); -} - -int main() -{ - context ctx; - - std::vector vA{3, 3, 2, 2, 17}; - plaintext pA(ctx, vA); - pA.set_symbol("A"); - - std::vector vB{1, 7, 7, 7, 49}; - plaintext pB(ctx, vB); - pB.set_symbol("B"); - - auto eA = pA.encrypt(); - auto eB = pB.encrypt(); - auto out = circuit(eA, eB); - - std::vector v_out; - out.decrypt().convert_to_vector(v_out); - - ctx.finalize(); - - for (size_t i = 0; i < v_out.size(); i++) - { - char expected = circuit(vA[i], vB[i]); - EXPECT(expected == v_out[i]); - } -} diff --git a/cccl_upstream/cudax/examples/stf/binary_fhe_stackable.cu b/cccl_upstream/cudax/examples/stf/binary_fhe_stackable.cu deleted file mode 100644 index 8ec91b48..00000000 --- a/cccl_upstream/cudax/examples/stf/binary_fhe_stackable.cu +++ /dev/null @@ -1,240 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief A toy example to illustrate how we can compose logical operations over encrypted data - */ - -#include - -using namespace cuda::experimental::stf; - -#include - -class ciphertext; - -class plaintext -{ -public: - plaintext(const stackable_ctx& ctx) - : ctx(ctx) - {} - - plaintext(stackable_ctx& ctx, ::std::vector v) - : values(mv(v)) - , ctx(ctx) - , ld(ctx.logical_data(values.data(), values.size())) - {} - - auto& set_symbol(const std::string& s) - { - ld.set_symbol(s); - symbol = s; - - return *this; - } - - const std::string& get_symbol() const - { - return symbol; - } - - // This will asynchronously fill string s - void convert_to_vector(std::vector& v) - { - ctx.host_launch(ld.read()).set_symbol("to_vector")->*[&](auto dl) { - v.resize(dl.size()); - for (size_t i = 0; i < dl.size(); i++) - { - v[i] = dl(i); - } - }; - } - - ciphertext encrypt() const; - -private: - std::vector values; - mutable stackable_ctx ctx; - ::std::string symbol; - -public: - mutable stackable_logical_data> ld; -}; - -class ciphertext -{ -public: - ciphertext() = default; - - // We need a deep-copy semantic - ciphertext(const ciphertext& other) - : ctx(other.ctx) - , symbol(other.symbol) - { - copy_content(ctx, other, *this); - } - - ciphertext(const stackable_ctx& ctx) - : ctx(ctx) - {} - - ciphertext(ciphertext&&) = default; - ciphertext& operator=(ciphertext&&) = default; - - static void copy_content(stackable_ctx& ctx, const ciphertext& src, ciphertext& dst) - { - dst.ld = ctx.logical_data(src.ld.shape()); - ctx.parallel_for(src.ld.shape(), src.ld.read(), dst.ld.write()).set_symbol("copy")->* - [] __device__(size_t i, auto src, auto dst) { - dst(i) = src(i); - }; - } - - auto& set_symbol(std::string s) - { - ld.set_symbol(s); - symbol = mv(s); - - return *this; - } - - const std::string& get_symbol() const - { - return symbol; - } - - plaintext decrypt() const - { - plaintext p(ctx); - p.ld = ctx.logical_data(shape_of>(ld.shape().size())); - ctx.parallel_for(ld.shape(), ld.read(), p.ld.write()).set_symbol("decrypt")->* - [] __device__(size_t i, auto cipher_data, auto plain_data) { - plain_data(i) = static_cast(cipher_data(i) >> 32); - }; - return p; - } - - // Copy assignment operator - // We need a deep-copy semantic - ciphertext& operator=(const ciphertext& other) - { - if (this != &other) - { - ctx = other.ctx; - symbol = other.symbol; - copy_content(ctx, other, *this); - } - return *this; - } - - ciphertext operator|(const ciphertext& other) const - { - ciphertext result(ctx); - result.ld = ctx.logical_data(ld.shape()); - - ctx.parallel_for(ld.shape(), ld.read(), other.ld.read(), result.ld.write()).set_symbol("OR")->* - [] __device__(size_t i, auto d_c1, auto d_c2, auto d_res) { - d_res(i) = d_c1(i) | d_c2(i); - }; - - return result; - } - - ciphertext operator&(const ciphertext& other) const - { - ciphertext result(ctx); - result.ld = ctx.logical_data(ld.shape()); - - ctx.parallel_for(ld.shape(), ld.read(), other.ld.read(), result.ld.write()).set_symbol("AND")->* - [] __device__(size_t i, auto d_c1, auto d_c2, auto d_res) { - d_res(i) = d_c1(i) & d_c2(i); - }; - - return result; - } - - ciphertext operator~() const - { - ciphertext result(ctx); - result.ld = ctx.logical_data(ld.shape()); - - ctx.parallel_for(ld.shape(), ld.read(), result.ld.write()).set_symbol("NOT")->* - [] __device__(size_t i, auto d_c, auto d_res) { - d_res(i) = ~d_c(i); - }; - - return result; - } - - mutable stackable_logical_data> ld; - -private: - mutable stackable_ctx ctx; - ::std::string symbol; -}; - -ciphertext plaintext::encrypt() const -{ - ciphertext c(ctx); - c.ld = ctx.logical_data(shape_of>(ld.shape().size())); - - ctx.parallel_for(ld.shape(), ld.read(), c.ld.write()).set_symbol("encrypt")->* - [] __device__(size_t i, auto dptxt, auto dctxt) { - // A super safe encryption ! - dctxt(i) = ((uint64_t) (dptxt(i)) << 32 | 0x4); - }; - - return c; -} - -template -T circuit(const T& a, const T& b) -{ - return ~((a | ~b) & (~a | b)); -} - -int main() -{ - stackable_ctx ctx; - - const std::vector vA{3, 3, 2, 2, 17}; - plaintext pA(ctx, std::vector(vA)); - pA.set_symbol("A"); - - const std::vector vB{1, 7, 7, 7, 49}; - plaintext pB(ctx, std::vector(vB)); - pB.set_symbol("B"); - - auto s_encrypt = ctx.dot_section("encrypt"); - auto eA = pA.encrypt().set_symbol("A"); - auto eB = pB.encrypt().set_symbol("B"); - s_encrypt.end(); - - ctx.push(); - - auto s_circuit = ctx.dot_section("circuit"); - auto out = circuit(eA, eB); - s_circuit.end(); - - ctx.pop(); - - std::vector v_out; - out.decrypt().convert_to_vector(v_out); - - ctx.finalize(); - - for (size_t i = 0; i < v_out.size(); i++) - { - char expected = circuit(vA[i], vB[i]); - EXPECT(expected == v_out[i]); - } -} diff --git a/cccl_upstream/cudax/examples/stf/cfd.cu b/cccl_upstream/cudax/examples/stf/cfd.cu deleted file mode 100644 index a4f8af01..00000000 --- a/cccl_upstream/cudax/examples/stf/cfd.cu +++ /dev/null @@ -1,480 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Simulation of a fluid over a regular grid with an implicit Jacobi solver - */ - -#include - -#include - -#include -#include -#include - -using namespace std::chrono; -using namespace cuda::experimental::stf; - -/* wall-clock time */ -double gettime() -{ - auto now = system_clock::now().time_since_epoch(); - return duration_cast>(now).count(); -} - -void writeplotfile(int m, int n, int scale) -{ - FILE* gnuplot = EXPECT(fopen("cfd.plt", "w")); - SCOPE(exit) - { - EXPECT(fclose(gnuplot) == 0); - }; - - fprintf(gnuplot, - "set terminal pngcairo\n" - "set output 'cfd_output.png'\n" - "set size square\n" - "set key off\n" - "unset xtics\n" - "unset ytics\n"); - - fprintf(gnuplot, "set xrange [%i:%i]\n", 1 - scale, m + scale); - fprintf(gnuplot, "set yrange [%i:%i]\n", 1 - scale, n + scale); - - fprintf(gnuplot, - "plot \"colourmap.dat\" w rgbimage, \"velocity.dat\" u " - "1:2:(%d*0.75*$3/sqrt($3**2+$4**2)):(%d*0.75*$4/sqrt($3**2+$4**2)) with vectors lc rgb \"#7F7F7F\"", - scale, - scale); - - // printf("\nWritten gnuplot script 'cfd.plt'\n"); -} - -double colfunc(double x) -{ - double x1 = 0.2; - double x2 = 0.5; - - double absx = fabs(x); - - if (absx > x2) - { - return 0.0; - } - else if (absx < x1) - { - return 1.0; - } - else - { - return 1.0 - pow((absx - x1) / (x2 - x1), 2); - } -} - -void hue2rgb(double hue, int& r, int& g, int& b) -{ - int rgbmax = 255; - - r = (int) (rgbmax * colfunc(hue - 1.0)); - g = (int) (rgbmax * colfunc(hue - 0.5)); - b = (int) (rgbmax * colfunc(hue)); -} - -void writedatafiles(context& ctx, logical_data> lpsi, int m, int n, int scale) -{ - auto lvel = ctx.logical_data(shape_of>(m, n, 2)).set_symbol("vel"); - auto lrgb = ctx.logical_data(shape_of>(m, n, 3)).set_symbol("rgb"); - - ctx.host_launch(lpsi.read(), lvel.write(), lrgb.write()).set_symbol("writedatafiles") - ->*[=](auto psi, auto vel, auto rgb) { - // printf("\n\nWriting data files ...\n"); - - // calculate velocities and hues - - for (int i = 0; i < m; i++) - { - for (int j = 0; j < n; j++) - { - vel(i, j, 0) = (psi(i + 1, j + 2) - psi(i + 1, j)) / 2.0; - vel(i, j, 1) = -(psi(i + 2, j + 1) - psi(i, j + 1)) / 2.0; - - double v1 = vel(i, j, 0); - double v2 = vel(i, j, 1); - - double modvsq = v1 * v1 + v2 * v2; - - double hue = pow(modvsq, 0.4); - - hue2rgb(hue, rgb(i, j, 0), rgb(i, j, 1), rgb(i, j, 2)); - } - } - - // write data - - FILE* cfile = EXPECT(fopen("colourmap.dat", "w")); - SCOPE(exit) - { - fclose(cfile); - }; - FILE* vfile = EXPECT(fopen("velocity.dat", "w")); - SCOPE(exit) - { - fclose(vfile); - }; - - for (int i = 0; i < m; i++) - { - int ix = i + 1; - - for (int j = 0; j < n; j++) - { - int iy = j + 1; - - fprintf(cfile, "%i %i %i %i %i\n", ix, iy, rgb(i, j, 0), rgb(i, j, 1), rgb(i, j, 2)); - - if ((ix - 1) % scale == (scale - 1) / 2 && (iy - 1) % scale == (scale - 1) / 2) - { - fprintf(vfile, "%i %i %f %f\n", ix, iy, vel(i, j, 0), vel(i, j, 1)); - } - } - } - - // printf("... done!\n"); - - writeplotfile(m, n, scale); - }; -} - -void jacobistep(context& ctx, logical_data> lpsinew, logical_data> lpsi, int m, int n) -{ - ctx.parallel_for(box<2>({1, m + 1}, {1, n + 1}), lpsinew.write(), lpsi.read()).set_symbol("jacobi_step") - ->*[] __device__(size_t i, size_t j, auto psinew, auto psi) { - psinew(i, j) = 0.25 * (psi(i - 1, j) + psi(i + 1, j) + psi(i, j + 1) + psi(i, j - 1)); - }; -} - -void jacobistepvort( - context& ctx, - logical_data> lzetnew, - logical_data> lpsinew, - logical_data> lzet, - logical_data> lpsi, - int m, - int n, - double re) -{ - ctx.parallel_for(box<2>({1, m + 1}, {1, n + 1}), lpsinew.write(), lpsi.read(), lzet.read()) - .set_symbol("jacobi_step_psi") - ->*[] __device__(size_t i, size_t j, auto psinew, auto psi, auto zet) { - psinew(i, j) = 0.25 * (psi(i - 1, j) + psi(i + 1, j) + psi(i, j + 1) + psi(i, j - 1)) - zet(i, j); - }; - - ctx.parallel_for(box<2>({1, m + 1}, {1, n + 1}), lzetnew.write(), lzet.read(), lpsi.read()) - .set_symbol("jacobi_step_zet") - ->*[=] __device__(size_t i, size_t j, auto zetnew, auto zet, auto psi) { - zetnew(i, j) = 0.25 * (zet(i - 1, j) + zet(i + 1, j) + zet(i, j + 1) + zet(i, j - 1)) - - re / 16.0 - * ((psi(i, j + 1) - psi(i, j - 1)) * (zet(i + 1, j) - zet(i - 1, j)) - - (psi(i + 1, j) - psi(i - 1, j)) * (zet(i, j + 1) - zet(i, j - 1))); - }; -} - -double deltasq(context& ctx, logical_data> lnewarr, logical_data> loldarr) -{ - auto ldsq = ctx.logical_data(shape_of>()).set_symbol("tmp_accumulator"); - - ctx.parallel_for(lnewarr.shape(), ldsq.reduce(reducer::sum{}), lnewarr.read(), loldarr.read()) - .set_symbol("deltasq") - ->*[] __device__(size_t i, size_t j, auto& dsq, auto newarr, auto oldarr) { - double tmp = newarr(i, j) - oldarr(i, j); - dsq += tmp * tmp; - }; - - return ctx.wait(ldsq); -} - -void boundarypsi(context& ctx, logical_data> lpsi, int m, int /*n*/, int b, int h, int w) -{ - // BCs on bottom edge - ctx.parallel_for(box({b + 1, b + w}), lpsi.rw()).set_symbol("boundary_bottom")->*[=] __device__(size_t i, auto psi) { - psi(i, 0) = double(i - b); - }; - - ctx.parallel_for(box<1>({b + w, m + 1}), lpsi.rw()).set_symbol("boundary_bottom")->*[=] __device__(size_t i, auto psi) { - psi(i, 0) = double(w); - }; - - // BCS on RHS - ctx.parallel_for(box({1, h + 1}), lpsi.rw()).set_symbol("boundary_right")->*[=] __device__(size_t j, auto psi) { - psi(m + 1, j) = double(w); - }; - - ctx.parallel_for(box({h + 1, h + w}), lpsi.rw()).set_symbol("boundary_right")->*[=] __device__(size_t j, auto psi) { - psi(m + 1, j) = (double) (w - j + h); - }; -} - -void boundaryzet(context& ctx, logical_data> lzet, logical_data> lpsi, int m, int n) -{ - // set top/bottom BCs: - ctx.parallel_for(box({1, m + 1}), lzet.rw(), lpsi.read()).set_symbol("boundary_topbottom") - ->*[=] __device__(size_t i, auto zet, auto psi) { - zet(i, 0) = 2.0 * (psi(i, 1) - psi(i, 0)); - zet(i, n + 1) = 2.0 * (psi(i, n) - psi(i, n + 1)); - }; - - // set left and right BCs: - ctx.parallel_for(box({1, n + 1}), lzet.rw(), lpsi.read()).set_symbol("boundary_leftright") - ->*[=] __device__(size_t j, auto zet, auto psi) { - zet(0, j) = 2.0 * (psi(1, j) - psi(0, j)); - zet(m + 1, j) = 2.0 * (psi(m, j) - psi(m + 1, j)); - }; -} - -int main(int argc, char** argv) -{ - context ctx; - - int printfreq = 10; // output frequency - double error = -1.0; - double tolerance = 0.0001; //-1.0; // 0.0001; //tolerance for convergence. <=0 means do not check - - // command line arguments - int scalefactor = 1, numiter = 10; - - double re = -1.0; // Reynold's number - must be less than 3.7 - - // simulation sizes - int bbase = 10; - int hbase = 15; - int wbase = 5; - int mbase = 32; - int nbase = 32; - - int irrotational = 1, checkerr = 0; - - // do we stop because of tolerance? - if (tolerance > 0) - { - checkerr = 1; - } - - // check command line parameters and parse them - - if (argc > 5) - { - printf("Usage: cfd [reynolds] [use_graphs]\n"); - return 0; - } - - if (argc > 1) - { - scalefactor = atoi(argv[1]); - } - - if (argc > 2) - { - numiter = atoi(argv[2]); - } - - if (argc > 3) - { - re = atof(argv[3]); - irrotational = 0; - } - - // Use a CUDA graph backend - if (argc > 4) - { - if (atoi(argv[4]) == 1) - { - ctx = graph_ctx(); - } - fprintf(stderr, "Using %s backend.\n", ctx.to_string().c_str()); - } - - // if (!checkerr) { - // printf("Scale Factor = %i, iterations = %i\n", scalefactor, numiter); - // } else { - // printf("Scale Factor = %i, iterations = %i, tolerance= %g\n", scalefactor, numiter, tolerance); - // } - - // if (irrotational) { - // printf("Irrotational flow\n"); - // } else { - // printf("Reynolds number = %f\n", re); - // } - - tolerance /= scalefactor; - - // Calculate b, h & w and m & n - int b = bbase * scalefactor; - int h = hbase * scalefactor; - int w = wbase * scalefactor; - int m = mbase * scalefactor; - int n = nbase * scalefactor; - - re /= scalefactor; - - // printf("Running CFD on %d x %d grid in serial\n", m, n); - - // main arrays and their temporary versions - logical_data> lzet, lzettmp, lpsi, lpsitmp; - - // allocate arrays - lpsi = ctx.logical_data(shape_of>(m + 2, n + 2)).set_symbol("psi"); - lpsitmp = ctx.logical_data(lpsi.shape()).set_symbol("psi_tmp"); - - // zero the psi array - ctx.parallel_for(lpsi.shape(), lpsi.write()).set_symbol("InitPsi")->*[] __device__(size_t i, size_t j, auto psi) { - psi(i, j) = 0.0; - }; - - if (!irrotational) - { - lzet = ctx.logical_data(lpsi.shape()).set_symbol("zet"); - lzettmp = ctx.logical_data(lpsi.shape()).set_symbol("zet_tmp"); - - // zero the zeta array - ctx.parallel_for(lzet.shape(), lzet.write()).set_symbol("InitZet")->*[] __device__(size_t i, size_t j, auto zet) { - zet(i, j) = 0.0; - }; - } - - // set the psi boundary conditions - boundarypsi(ctx, lpsi, m, n, b, h, w); - - // compute normalisation factor for error - auto lbnorm = ctx.logical_data(shape_of>()).set_symbol("bnorm"); - - nvtx_range r_norm("Compute_Normalization"); - - // bnorm = psi * psi - ctx.parallel_for(lpsi.shape(), lpsi.read(), lbnorm.reduce(reducer::sum{})) - ->*[] __device__(size_t i, size_t j, auto psi, auto& bnorm) { - bnorm += psi(i, j) * psi(i, j); - }; - - if (!irrotational) - { - // update zeta BCs that depend on psi - boundaryzet(ctx, lzet, lpsi, m, n); - - // update normalisation - ctx.parallel_for(lzet.shape(), lzet.read(), lbnorm.reduce(reducer::sum{}, no_init{})) - ->*[] __device__(size_t i, size_t j, auto zet, auto& bnorm_zet) { - bnorm_zet += zet(i, j) * zet(i, j); - }; - } - - r_norm.end(); - - double bnorm = ctx.wait(lbnorm); - bnorm = sqrt(bnorm); - - // begin iterative Jacobi loop - - // printf("\nStarting main loop...\n\n"); - - double tstart = gettime(); - nvtx_range r_iter("Overall_Iteration"); - - int iter = 1; - for (; iter <= numiter; iter++) - { - // calculate psi for next iteration - if (irrotational) - { - jacobistep(ctx, lpsitmp, lpsi, m, n); - } - else - { - jacobistepvort(ctx, lzettmp, lpsitmp, lzet, lpsi, m, n, re); - } - - // calculate current error if required - bool compute_error = (iter == numiter) || (checkerr && (iter % printfreq == 0)); - if (compute_error) - { - error = deltasq(ctx, lpsitmp, lpsi); - - if (!irrotational) - { - error += deltasq(ctx, lzettmp, lzet); - } - - error = sqrt(error); - error = error / bnorm; - - if (checkerr && (error < tolerance)) - { - // printf("Converged on iteration %d\n", iter); - break; - } - } - - // copy back - ctx.parallel_for(box<2>({1, m + 1}, {1, n + 1}), lpsi.rw(), lpsitmp.read()).set_symbol("SwitchPsi") - ->*[] __device__(size_t i, size_t j, auto psi, auto psitmp) { - psi(i, j) = psitmp(i, j); - }; - - if (!irrotational) - { - ctx.parallel_for(box<2>({1, m + 1}, {1, n + 1}), lzet.rw(), lzettmp.read()).set_symbol("SwitchZet") - ->*[] __device__(size_t i, size_t j, auto zet, auto zettmp) { - zet(i, j) = zettmp(i, j); - }; - } - - if (!irrotational) - { - // update zeta BCs that depend on psi - boundaryzet(ctx, lzet, lpsi, m, n); - } - - // if (iter % printfreq == 0) { - // if (!checkerr) { - // printf("Completed iteration %d\n", iter); - // } else { - // printf("Completed iteration %d, error = %g\n", iter, error); - // } - // } - } - r_iter.end(); - - if (iter > numiter) - { - iter = numiter; - } - - double tstop = gettime(); - - double ttot = tstop - tstart; - double titer = ttot / (double) iter; - - // output results - - writedatafiles(ctx, lpsi, m, n, scalefactor); - - ctx.finalize(); - // print out some stats - - // printf("\n... finished\n"); - printf("After %d iterations, the error is %g\n", iter, error); - printf("Time for %d iterations was %g seconds\n", iter, ttot); - printf("Each iteration took %g seconds\n", titer); - - // printf("... finished\n"); - - return 0; -} diff --git a/cccl_upstream/cudax/examples/stf/custom_data_interface.cu b/cccl_upstream/cudax/examples/stf/custom_data_interface.cu deleted file mode 100644 index 15bb40ce..00000000 --- a/cccl_upstream/cudax/examples/stf/custom_data_interface.cu +++ /dev/null @@ -1,327 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief This example illustrates how to create a custom data interface and use them in tasks - * - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -/** - * @brief A simple class describing a contiguous matrix of size (m, n) - */ -template -class matrix -{ -public: - matrix(size_t m, size_t n, T* base) - : m(m) - , n(n) - , base(base) - {} - - __host__ __device__ T& operator()(size_t i, size_t j) - { - return base[i + j * m]; - } - - __host__ __device__ const T& operator()(size_t i, size_t j) const - { - return base[i + j * m]; - } - - size_t m, n; - T* base; -}; - -/** - * @brief defines the shape of a matrix - * - * Note that we specialize cuda::experimental::stf::shape_of to avoid ambiguous specialization - * - * @extends shape_of - */ -template -class cuda::experimental::stf::shape_of> -{ -public: - /** - * @brief The default constructor. - * - * All `shape_of` specializations must define this constructor. - */ - shape_of() = default; - - explicit shape_of(size_t m, size_t n) - : m(m) - , n(n) - {} - - /** - * @name Copies a shape. - * - * All `shape_of` specializations must define this constructor. - */ - shape_of(const shape_of&) = default; - - /** - * @brief Extracts the shape from a matrix - * - * @param M matrix to get the shape from - * - * All `shape_of` specializations must define this constructor. - */ - shape_of(const matrix& M) - : shape_of>(M.m, M.n) - {} - - /// Mandatory method : defined the total number of elements in the shape - size_t size() const - { - return m * n; - } - - using coords_t = ::cuda::std::array; - - // This transforms a tuple of (shape, 1D index) into a coordinate - _CCCL_HOST_DEVICE coords_t index_to_coords(size_t index) const - { - return {index % m, index / m}; - } - - size_t m; - size_t n; -}; - -/** - * @brief Data interface to manipulate a matrix in the CUDA stream backend - */ -template -class matrix_stream_interface : public stream_data_interface_simple> -{ -public: - using base = stream_data_interface_simple>; - using typename base::shape_t; - - /// Initialize from an existing matrix - matrix_stream_interface(matrix m) - : base(std::move(m)) - {} - - /// Initialize from a shape of matrix - matrix_stream_interface(typename base::shape_t s) - : base(s) - {} - - /// Copy the content of an instance to another instance - /// - /// This implementation assumes that we have registered memory if one of the data place is the host - void stream_data_copy( - const data_place& dst_memory_node, - instance_id_t dst_instance_id, - const data_place& src_memory_node, - instance_id_t src_instance_id, - cudaStream_t stream) override - { - assert(src_memory_node != dst_memory_node); - - cudaMemcpyKind kind = cudaMemcpyDeviceToDevice; - if (src_memory_node.is_host()) - { - kind = cudaMemcpyHostToDevice; - } - - if (dst_memory_node.is_host()) - { - kind = cudaMemcpyDeviceToHost; - } - - const matrix& src_instance = this->instance(src_instance_id); - const matrix& dst_instance = this->instance(dst_instance_id); - - size_t sz = src_instance.m * src_instance.n * sizeof(T); - - cuda_safe_call(cudaMemcpyAsync((void*) dst_instance.base, (void*) src_instance.base, sz, kind, stream)); - } - - /// allocate an instance on a specific data place - /// - /// setting *s to a negative value informs CUDASTF that the allocation - /// failed, and that a memory reclaiming mechanism need to be performed. - void stream_data_allocate( - backend_ctx_untyped& /*unused*/, - const data_place& memory_node, - instance_id_t instance_id, - ::std::ptrdiff_t& s, - void** /*unused*/, - cudaStream_t stream) override - { - matrix& instance = this->instance(instance_id); - size_t sz = instance.m * instance.n * sizeof(T); - - T* base_ptr; - - if (memory_node.is_host()) - { - // Fallback to a synchronous method as there is no asynchronous host allocation API - cuda_safe_call(cudaStreamSynchronize(stream)); - cuda_safe_call(cudaHostAlloc(&base_ptr, sz, cudaHostAllocMapped)); - } - else - { - cuda_safe_call(cudaMallocAsync(&base_ptr, sz, stream)); - } - - // By filling a positive number, we notify that the allocation was successful - s = sz; - - instance.base = base_ptr; - } - - /// deallocate an instance - void stream_data_deallocate( - backend_ctx_untyped& /*unused*/, - const data_place& memory_node, - instance_id_t instance_id, - void* /*unused*/, - cudaStream_t stream) override - { - matrix& instance = this->instance(instance_id); - if (memory_node.is_host()) - { - // Fallback to a synchronous method as there is no asynchronous host deallocation API - cuda_safe_call(cudaStreamSynchronize(stream)); - cuda_safe_call(cudaFreeHost(instance.base)); - } - else - { - cuda_safe_call(cudaFreeAsync(instance.base, stream)); - } - } - - /// Register the host memory associated to an instance of matrix - /// - /// Note that this pin_host_memory method is not mandatory, but then it is - /// the responsibility of the user to only passed memory that is already - /// registered, and the allocation method on the host must allocate - /// registered memory too. Otherwise, copy methods need to be synchronous. - bool pin_host_memory(instance_id_t instance_id) override - { - matrix& instance = this->instance(instance_id); - if (!instance.base) - { - return false; - } - - cuda_safe_call(pin_memory(instance.base, instance.m * instance.n * sizeof(T))); - - return true; - } - - /// Unregister memory pinned by pin_host_memory - void unpin_host_memory(instance_id_t instance_id) override - { - matrix& instance = this->instance(instance_id); - unpin_memory(instance.base); - } -}; - -/** - * @brief Define how the CUDA stream backend must manipulate a matrix - * - * Note that we specialize cuda::experimental::stf::shape_of to avoid ambiguous specialization - * - * @extends streamed_interface_of - */ -template -struct cuda::experimental::stf::streamed_interface_of> -{ - using type = matrix_stream_interface; -}; - -/** - * @brief A hash of the matrix - */ -template -struct cuda::experimental::stf::hash> -{ - std::size_t operator()(matrix const& m) const noexcept - { - // Combine hashes from the base address and sizes - return cuda::experimental::stf::hash_all(m.m, m.n, m.base); - } -}; - -template -__global__ void kernel(matrix M) -{ - int tid_x = blockIdx.x * blockDim.x + threadIdx.x; - int nthreads_x = gridDim.x * blockDim.x; - - int tid_y = blockIdx.y * blockDim.y + threadIdx.y; - int nthreads_y = gridDim.y * blockDim.y; - - for (int x = tid_x; x < M.m; x += nthreads_x) - { - for (int y = tid_y; y < M.n; y += nthreads_y) - { - M(x, y) += -x + 7 * y; - } - } -} - -int main() -{ - stream_ctx ctx; - - const size_t m = 8; - const size_t n = 10; - std::vector v(m * n); - - matrix M(m, n, &v[0]); - - // M(i,j) = 17 * i + 23 * j - for (size_t j = 0; j < n; j++) - { - for (size_t i = 0; i < m; i++) - { - M(i, j) = 17 * i + 23 * j; - } - } - - auto lM = ctx.logical_data(M); - - // M(i,j) += -i + 7*i - ctx.task(lM.rw())->*[](cudaStream_t s, auto dM) { - kernel<<>>(dM); - }; - - // M(i,j) += 2*i + 6*j - ctx.parallel_for(lM.shape(), lM.rw())->*[] _CCCL_DEVICE(size_t i, size_t j, auto dM) { - dM(i, j) += 2 * i + 6 * j; - }; - - ctx.finalize(); - - for (size_t j = 0; j < n; j++) - { - for (size_t i = 0; i < m; i++) - { - assert(M(i, j) == (17 * i + 23 * j) + (-i + 7 * j) + (2 * i + 6 * j)); - } - } -} diff --git a/cccl_upstream/cudax/examples/stf/explicit_data_places.cu b/cccl_upstream/cudax/examples/stf/explicit_data_places.cu deleted file mode 100644 index f8c53e31..00000000 --- a/cccl_upstream/cudax/examples/stf/explicit_data_places.cu +++ /dev/null @@ -1,87 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief An AXPY kernel implemented with a task of the CUDA stream backend - * where the task accesses host memory from the device - * - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -__global__ void axpy(double a, slice x, slice y) -{ - int tid = blockIdx.x * blockDim.x + threadIdx.x; - int nthreads = gridDim.x * blockDim.x; - - for (int i = tid; i < x.size(); i += nthreads) - { - y(i) += a * x(i); - } -} - -double X0(int i) -{ - return sin((double) i); -} - -double Y0(int i) -{ - return cos((double) i); -} - -int main() -{ - // Verify whether this device can access memory concurrently from CPU and GPU. - int dev; - cuda_safe_call(cudaGetDevice(&dev)); - assert(dev >= 0); - cudaDeviceProp prop; - cuda_safe_call(cudaGetDeviceProperties(&prop, dev)); - if (!prop.concurrentManagedAccess) - { - fprintf(stderr, "Concurrent CPU/GPU access not supported, skipping test.\n"); - return 0; - } - - stream_ctx ctx; - const size_t N = 16; - double X[N], Y[N]; - - for (size_t i = 0; i < N; i++) - { - X[i] = X0(i); - Y[i] = Y0(i); - } - - double alpha = 3.14; - - auto lX = ctx.logical_data(X); - auto lY = ctx.logical_data(Y); - - /* Compute Y = Y + alpha X, but leave X on the host and access it with mapped memory */ - ctx.task(lX.read(data_place::host()), lY.rw())->*[&](cudaStream_t s, auto dX, auto dY) { - axpy<<<16, 128, 0, s>>>(alpha, dX, dY); - }; - - ctx.finalize(); - - for (size_t i = 0; i < N; i++) - { - assert(fabs(Y[i] - (Y0(i) + alpha * X0(i))) < 0.0001); - assert(fabs(X[i] - X0(i)) < 0.0001); - } -} diff --git a/cccl_upstream/cudax/examples/stf/fdtd_mgpu.cu b/cccl_upstream/cudax/examples/stf/fdtd_mgpu.cu deleted file mode 100644 index b65617b4..00000000 --- a/cccl_upstream/cudax/examples/stf/fdtd_mgpu.cu +++ /dev/null @@ -1,300 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief An example solving Maxwell equations in 3D using FDTD on multiple devices - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -// FIXME : MSVC has trouble with box constructors -#if !_CCCL_COMPILER(MSVC) -void write_vtk_2D(const std::string& filename, slice Ez, double dx, double dy, double /*unused*/) -{ - FILE* f = fopen(filename.c_str(), "w"); - - const size_t pos_z = Ez.extent(2) / 2; - const size_t nx = Ez.extent(0); - - const size_t size = Ez.extent(0) * Ez.extent(1); - - fprintf(f, "# vtk DataFile Version 3.0\n"); - fprintf(f, "vtk output\n"); - fprintf(f, "ASCII\n"); - fprintf(f, "DATASET UNSTRUCTURED_GRID\n"); - fprintf(f, "POINTS %ld float\n", 4 * size); - - for (size_t y = 0; y < Ez.extent(1); y++) - { - for (size_t x = 0; x < Ez.extent(0); x++) - { - fprintf(f, "%lf %lf 0.0\n", dx * static_cast(x + 0), dy * static_cast(y + 0)); - fprintf(f, "%lf %lf 0.0\n", dx * static_cast(x + 1), dy * static_cast(y + 0)); - fprintf(f, "%lf %lf 0.0\n", dx * static_cast(x + 1), dy * static_cast(y + 1)); - fprintf(f, "%lf %lf 0.0\n", dx * static_cast(x + 0), dy * static_cast(y + 1)); - } - } - - fprintf(f, "CELLS %ld %ld\n", size, 5 * size); - - size_t cell_id = 0; - for (size_t y = 0; y < Ez.extent(1); y++) - { - for (size_t x = 0; x < Ez.extent(0); x++) - { - const size_t point_offset = cell_id * 4; - fprintf(f, - "4 %d %d %d %d\n", - (int) (point_offset + 0), - (int) (point_offset + 1), - (int) (point_offset + 2), - (int) (point_offset + 3)); - - cell_id++; - } - } - - fprintf(f, "CELL_TYPES %ld\n", size); - - for (size_t ii = 0; ii < size; ii++) - { - fprintf(f, "5\n"); - } - - fprintf(f, "CELL_DATA %ld\n", size); - fprintf(f, "SCALARS Ez double 1\n"); - fprintf(f, "LOOKUP_TABLE default\n"); - - for (size_t y = 0; y < Ez.extent(1); y++) - { - for (size_t x = 0; x < Ez.extent(0); x++) - { - fprintf(f, "%lf\n", Ez(x, y, pos_z)); - } - } - - fclose(f); -} - -// Define the source function -_CCCL_DEVICE double Source(double t, double x, double y, double z) -{ - constexpr double pi = 3.14159265358979323846; - constexpr double freq = 1e9; - constexpr double omega = (2 * pi * freq); - constexpr double wavelength = 3e8 / freq; - constexpr double k = 2 * pi / wavelength; - return sin(k * x - omega * t); -} -#endif // !_CCCL_COMPILER(MSVC) - -int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) -{ -#if !_CCCL_COMPILER(MSVC) - context ctx; - - // Initialize the time loop - size_t timesteps = 10; - if (argc > 1) - { - timesteps = (size_t) atol(argv[1]); - } - - // No output by default - int output_freq = -1; - if (argc > 2) - { - output_freq = atoi(argv[2]); - } - - // Default value : grid of all devices - exec_place where = exec_place::all_devices(); - - if (argc > 3) - { - switch (atoi(argv[3])) - { - case 0: - where = exec_place::host(); - break; - case 1: - where = exec_place::current_device(); - break; - case 2: - where = exec_place::all_devices(); - break; - case 3: - where = exec_place::repeat(exec_place::current_device(), 8); - break; - default: - fprintf(stderr, "Invalid exec place argument\n"); - abort(); - } - - fprintf(stderr, "Running on %s\n", where.to_string().c_str()); - } - - if (argc > 4) - { - int use_graph = atoi(argv[4]); - if (use_graph) - { - ctx = graph_ctx(); - } - - fprintf(stderr, "Use %s backend.\n", use_graph ? "graph" : "stream"); - } - - // Domain dimensions - const size_t SIZE_X = 100; - const size_t SIZE_Y = 100; - const size_t SIZE_Z = 100; - - // Grid spacing - const double DX = 0.01; - const double DY = 0.01; - const double DZ = 0.01; - - // Define the electric and magnetic fields - auto data_shape = shape_of>(SIZE_X, SIZE_Y, SIZE_Z); - - // One structured partition drives every task's decomposition AND the data - // placement: dimension 2 blocked over the grid of devices (change the spec - // entry to split any other dimension). Interior boxes below iterate each - // place's owned coordinates restricted to the box, and uneven sizes are - // handled by predication. - auto part = make_partition(dim4(SIZE_X, SIZE_Y, SIZE_Z), partition_spec{whole, whole, blocked<0>}, where.get_dims()); - auto lEx = ctx.logical_data(data_shape); - auto lEy = ctx.logical_data(data_shape); - auto lEz = ctx.logical_data(data_shape); - auto lHx = ctx.logical_data(data_shape); - auto lHy = ctx.logical_data(data_shape); - auto lHz = ctx.logical_data(data_shape); - - // Define the permittivity and permeability of the medium - auto lepsilon = ctx.logical_data(data_shape); - auto lmu = ctx.logical_data(data_shape); - - const double EPSILON = 8.85e-12; // Permittivity of free space - const double MU = 1.256e-6; // Permeability of free space - - // CFL condition DT <= min(DX, DY, DZ) * sqrt(epsilon_max * mu_max) - double DT = 0.25 * min(min(DX, DY), DZ) * sqrt(EPSILON * MU); - - // Initialize E - ctx.parallel_for(part, where, data_shape, lEx.write(), lEy.write(), lEz.write()) - ->*[] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Ex, auto Ey, auto Ez) { - Ex(i, j, k) = 0.0; - Ey(i, j, k) = 0.0; - Ez(i, j, k) = 0.0; - }; - - // Initialize H - ctx.parallel_for(part, where, data_shape, lHx.write(), lHy.write(), lHz.write()) - ->*[] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Hx, auto Hy, auto Hz) { - Hx(i, j, k) = 0.0; - Hy(i, j, k) = 0.0; - Hz(i, j, k) = 0.0; - }; - - // Initialize permittivity and permeability fields - ctx.parallel_for(part, where, data_shape, lepsilon.write(), lmu.write()) - ->*[=] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto epsilon, auto mu) { - epsilon(i, j, k) = EPSILON; - mu(i, j, k) = MU; - }; - - // Set the source function at the center of the grid - - const size_t center_x = SIZE_X / 2; - const size_t center_y = SIZE_Y / 2; - const size_t center_z = SIZE_Z / 2; - - // Index shapes for the electric and magnetic fields - box Es({1ul, SIZE_X - 1}, {1ul, SIZE_Y - 1}, {1ul, SIZE_Z - 1}); - box Hs({0ul, SIZE_X - 1}, {0ul, SIZE_Y - 1}, {0ul, SIZE_Z - 1}); - - ctx.repeat(timesteps)->*[&](context ctx, size_t n) { - // Update the electric fields - - // Update Ex - ctx.parallel_for(part, where, Es, lEx.rw(), lHy.read(), lHz.read(), lepsilon.read()) - ->*[=] - _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Ex, auto Hy, auto Hz, auto epsilon) { - Ex(i, j, k) = Ex(i, j, k) - + (DT / (epsilon(i, j, k) * DX)) * (Hz(i, j, k) - Hz(i, j - 1, k) - Hy(i, j, k) + Hy(i, j, k - 1)); - }; - - // Update Ey - ctx.parallel_for(part, where, Es, lEy.rw(), lHx.read(), lHz.read(), lepsilon.read()) - ->*[=] - _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Ey, auto Hx, auto Hz, auto epsilon) { - Ey(i, j, k) = Ey(i, j, k) - + (DT / (epsilon(i, j, k) * DY)) * (Hx(i, j, k) - Hx(i, j, k - 1) - Hz(i, j, k) + Hz(i - 1, j, k)); - }; - - // Update Ez and inject the point source in the same volumetric pass - ctx.parallel_for(part, where, Es, lEz.rw(), lHx.read(), lHy.read(), lepsilon.read()) - ->*[=] - _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Ez, auto Hx, auto Hy, auto epsilon) { - Ez(i, j, k) = Ez(i, j, k) - + (DT / (epsilon(i, j, k) * DZ)) * (Hy(i, j, k) - Hy(i - 1, j, k) - Hx(i, j, k) + Hx(i, j - 1, k)); - if (i == center_x && j == center_y && k == center_z) - { - Ez(i, j, k) += Source(n * DT, i * DX, j * DY, k * DZ); - } - }; - - // Update the magnetic fields - - // Update Hx - ctx.parallel_for(part, where, Hs, lHx.rw(), lEy.read(), lEz.read(), lmu.read()) - ->*[=] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Hx, auto Ey, auto Ez, auto mu) { - Hx(i, j, k) = Hx(i, j, k) - - (DT / (mu(i, j, k) * DY)) * (Ez(i, j + 1, k) - Ez(i, j, k) - Ey(i, j, k + 1) + Ey(i, j, k)); - }; - - // Update Hy - ctx.parallel_for(part, where, Hs, lHy.rw(), lEx.read(), lEz.read(), lmu.read()) - ->*[=] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Hy, auto Ex, auto Ez, auto mu) { - Hy(i, j, k) = Hy(i, j, k) - - (DT / (mu(i, j, k) * DZ)) * (Ex(i, j, k + 1) - Ex(i, j, k) - Ez(i + 1, j, k) + Ez(i, j, k)); - }; - - // Update Hz - ctx.parallel_for(part, where, Hs, lHz.rw(), lEx.read(), lEy.read(), lmu.read()) - ->*[=] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Hz, auto Ex, auto Ey, auto mu) { - Hz(i, j, k) = Hz(i, j, k) - - (DT / (mu(i, j, k) * DX)) * (Ey(i + 1, j, k) - Ey(i, j, k) - Ex(i, j + 1, k) + Ex(i, j, k)); - }; - - if (output_freq > 0 && n % output_freq == 0) - { - ctx.host_launch(lEz.read())->*[=](auto Ez) { - // Output the electric field at the center of the grid - fprintf(stderr, "%ld\t%le\n", n, Ez(center_x, center_y, center_z)); - - std::string filename = "Ez" + std::to_string(n) + ".vtk"; - - // Dump a 2D slice of Ez in VTK - write_vtk_2D(filename, Ez, DX, DY, DZ); - }; - } - }; - - ctx.finalize(); -#endif // !_CCCL_COMPILER(MSVC) -} diff --git a/cccl_upstream/cudax/examples/stf/fdtd_repeat_n.cu b/cccl_upstream/cudax/examples/stf/fdtd_repeat_n.cu deleted file mode 100644 index 6bd27629..00000000 --- a/cccl_upstream/cudax/examples/stf/fdtd_repeat_n.cu +++ /dev/null @@ -1,186 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief FDTD example using the repeat_n helper function - * - * This shows how to refactor the original fdtd_while.cu example - * to use the new repeat_n helper for cleaner loop patterns. - */ - -#include - -#include - -#include - -using namespace cuda::experimental::stf; - -// Define the source function -_CCCL_DEVICE double Source(double t, double x, double y, double z) -{ - constexpr double pi = 3.14159265358979323846; - constexpr double freq = 1e9; - constexpr double omega = (2 * pi * freq); - constexpr double wavelength = 3e8 / freq; - constexpr double k = 2 * pi / wavelength; - return sin(k * x - omega * t); -} - -int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) -{ -#if _CCCL_CTK_BELOW(12, 4) - fprintf(stderr, "Waiving test: conditional nodes are only available since CUDA 12.4.\n"); - return 0; -#else - stackable_ctx ctx; - - // Initialize the time loop - size_t timesteps = 10; - if (argc > 1) - { - timesteps = (size_t) atol(argv[1]); - } - - // Domain dimensions (smaller for this example) - const size_t SIZE_X = 50; - const size_t SIZE_Y = 50; - const size_t SIZE_Z = 50; - - // Grid spacing - const double DX = 0.01; - const double DY = 0.01; - const double DZ = 0.01; - - // Define the electric and magnetic fields - auto data_shape = shape_of>(SIZE_X, SIZE_Y, SIZE_Z); - auto lEx = ctx.logical_data(data_shape); - auto lEy = ctx.logical_data(data_shape); - auto lEz = ctx.logical_data(data_shape); - auto lHx = ctx.logical_data(data_shape); - auto lHy = ctx.logical_data(data_shape); - auto lHz = ctx.logical_data(data_shape); - - // Define the permittivity and permeability of the medium - auto lepsilon = ctx.logical_data(data_shape); - auto lmu = ctx.logical_data(data_shape); - - const double EPSILON = 8.85e-12; // Permittivity of free space - const double MU = 1.256e-6; // Permeability of free space - - // CFL condition DT <= min(DX, DY, DZ) * sqrt(epsilon_max * mu_max) - double DT = 0.25 * min(min(DX, DY), DZ) * sqrt(EPSILON * MU); - - // Initialize E fields - ctx.parallel_for(data_shape, lEx.write(), lEy.write(), lEz.write()) - ->*[] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Ex, auto Ey, auto Ez) { - Ex(i, j, k) = 0.0; - Ey(i, j, k) = 0.0; - Ez(i, j, k) = 0.0; - }; - - // Initialize H fields - ctx.parallel_for(data_shape, lHx.write(), lHy.write(), lHz.write()) - ->*[] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Hx, auto Hy, auto Hz) { - Hx(i, j, k) = 0.0; - Hy(i, j, k) = 0.0; - Hz(i, j, k) = 0.0; - }; - - // Initialize permittivity and permeability fields - ctx.parallel_for(data_shape, lepsilon.write(), lmu.write()) - ->*[=] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto epsilon, auto mu) { - epsilon(i, j, k) = EPSILON; - mu(i, j, k) = MU; - }; - - // Set the source location - const size_t center_x = SIZE_X / 2; - const size_t center_y = SIZE_Y / 2; - const size_t center_z = SIZE_Z / 2; - - // Index shapes for Electric fields, Magnetic fields, and the source - box Es({1ul, SIZE_X - 1}, {1ul, SIZE_Y - 1}, {1ul, SIZE_Z - 1}); - box Hs({0ul, SIZE_X - 1}, {0ul, SIZE_Y - 1}, {0ul, SIZE_Z - 1}); - box source_s({center_x, center_x + 1}, {center_y, center_y + 1}, {center_z, center_z + 1}); - - std::cout << "Running FDTD simulation for " << timesteps << " timesteps" << '\n'; - std::cout << "Grid size: " << SIZE_X << "x" << SIZE_Y << "x" << SIZE_Z << '\n'; - - { - auto repeat_guard = ctx.repeat_graph_scope(timesteps); - - // Update Ex - ctx.parallel_for(Es, lEx.rw(), lHy.read(), lHz.read(), lepsilon.read()) - ->*[=] - _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Ex, auto Hy, auto Hz, auto epsilon) { - Ex(i, j, k) = Ex(i, j, k) - + (DT / (epsilon(i, j, k) * DX)) * (Hz(i, j, k) - Hz(i, j - 1, k) - Hy(i, j, k) + Hy(i, j, k - 1)); - }; - - // Update Ey - ctx.parallel_for(Es, lEy.rw(), lHx.read(), lHz.read(), lepsilon.read()) - ->*[=] - _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Ey, auto Hx, auto Hz, auto epsilon) { - Ey(i, j, k) = Ey(i, j, k) - + (DT / (epsilon(i, j, k) * DY)) * (Hx(i, j, k) - Hx(i, j, k - 1) - Hz(i, j, k) + Hz(i - 1, j, k)); - }; - - // Update Ez - ctx.parallel_for(Es, lEz.rw(), lHx.read(), lHy.read(), lepsilon.read()) - ->*[=] - _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Ez, auto Hx, auto Hy, auto epsilon) { - Ez(i, j, k) = Ez(i, j, k) - + (DT / (epsilon(i, j, k) * DZ)) * (Hy(i, j, k) - Hy(i - 1, j, k) - Hx(i, j, k) + Hx(i, j - 1, k)); - }; - - // Add the source function at the center of the grid - // Note: We could add a current iteration tracker if needed for time-dependent sources - ctx.parallel_for(source_s, lEz.rw())->*[=] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Ez) { - // For simplicity, using a constant source in this example - // In the full version, you'd want to track the current timestep - Ez(i, j, k) = Ez(i, j, k) + 0.1 * sin(0.1 * (i + j + k)); - }; - - // Update Hx - ctx.parallel_for(Hs, lHx.rw(), lEy.read(), lEz.read(), lmu.read()) - ->*[=] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Hx, auto Ey, auto Ez, auto mu) { - Hx(i, j, k) = Hx(i, j, k) - - (DT / (mu(i, j, k) * DY)) * (Ez(i, j + 1, k) - Ez(i, j, k) - Ey(i, j, k + 1) + Ey(i, j, k)); - }; - - // Update Hy - ctx.parallel_for(Hs, lHy.rw(), lEx.read(), lEz.read(), lmu.read()) - ->*[=] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Hy, auto Ex, auto Ez, auto mu) { - Hy(i, j, k) = Hy(i, j, k) - - (DT / (mu(i, j, k) * DZ)) * (Ex(i, j, k + 1) - Ex(i, j, k) - Ez(i + 1, j, k) + Ez(i, j, k)); - }; - - // Update Hz - ctx.parallel_for(Hs, lHz.rw(), lEx.read(), lEy.read(), lmu.read()) - ->*[=] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Hz, auto Ex, auto Ey, auto mu) { - Hz(i, j, k) = Hz(i, j, k) - - (DT / (mu(i, j, k) * DX)) * (Ey(i + 1, j, k) - Ey(i, j, k) - Ex(i, j + 1, k) + Ex(i, j, k)); - }; - } // repeat_guard - - // Print final result at center - ctx.host_launch(lEz.read())->*[=](auto Ez) { - std::cout << "Final Ez at center: " << Ez(center_x, center_y, center_z) << '\n'; - }; - - ctx.finalize(); - - std::cout << "FDTD simulation completed!" << '\n'; - return 0; -#endif -} diff --git a/cccl_upstream/cudax/examples/stf/fdtd_while.cu b/cccl_upstream/cudax/examples/stf/fdtd_while.cu deleted file mode 100644 index 6f15d42c..00000000 --- a/cccl_upstream/cudax/examples/stf/fdtd_while.cu +++ /dev/null @@ -1,288 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief An example solving Maxwell equations in 3D using FDTD on multiple devices - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -// FIXME : MSVC has trouble with box constructors -#if !_CCCL_COMPILER(MSVC) -void write_vtk_2D(const std::string& filename, slice Ez, double dx, double dy, double /*unused*/) -{ - FILE* f = fopen(filename.c_str(), "w"); - - const size_t pos_z = Ez.extent(2) / 2; - const size_t nx = Ez.extent(0); - - const size_t size = Ez.extent(0) * Ez.extent(1); - - fprintf(f, "# vtk DataFile Version 3.0\n"); - fprintf(f, "vtk output\n"); - fprintf(f, "ASCII\n"); - fprintf(f, "DATASET UNSTRUCTURED_GRID\n"); - fprintf(f, "POINTS %ld float\n", 4 * size); - - for (size_t y = 0; y < Ez.extent(1); y++) - { - for (size_t x = 0; x < Ez.extent(0); x++) - { - fprintf(f, "%lf %lf 0.0\n", dx * static_cast(x + 0), dy * static_cast(y + 0)); - fprintf(f, "%lf %lf 0.0\n", dx * static_cast(x + 1), dy * static_cast(y + 0)); - fprintf(f, "%lf %lf 0.0\n", dx * static_cast(x + 1), dy * static_cast(y + 1)); - fprintf(f, "%lf %lf 0.0\n", dx * static_cast(x + 0), dy * static_cast(y + 1)); - } - } - - fprintf(f, "CELLS %ld %ld\n", size, 5 * size); - - size_t cell_id = 0; - for (size_t y = 0; y < Ez.extent(1); y++) - { - for (size_t x = 0; x < Ez.extent(0); x++) - { - const size_t point_offset = cell_id * 4; - fprintf(f, - "4 %d %d %d %d\n", - (int) (point_offset + 0), - (int) (point_offset + 1), - (int) (point_offset + 2), - (int) (point_offset + 3)); - - cell_id++; - } - } - - fprintf(f, "CELL_TYPES %ld\n", size); - - for (size_t ii = 0; ii < size; ii++) - { - fprintf(f, "5\n"); - } - - fprintf(f, "CELL_DATA %ld\n", size); - fprintf(f, "SCALARS Ez double 1\n"); - fprintf(f, "LOOKUP_TABLE default\n"); - - for (size_t y = 0; y < Ez.extent(1); y++) - { - for (size_t x = 0; x < Ez.extent(0); x++) - { - fprintf(f, "%lf\n", Ez(x, y, pos_z)); - } - } - - fclose(f); -} - -// Define the source function -_CCCL_DEVICE double Source(double t, double x, double y, double z) -{ - constexpr double pi = 3.14159265358979323846; - constexpr double freq = 1e9; - constexpr double omega = (2 * pi * freq); - constexpr double wavelength = 3e8 / freq; - constexpr double k = 2 * pi / wavelength; - return sin(k * x - omega * t); -} -#endif // !_CCCL_COMPILER(MSVC) - -int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) -{ -#if !_CCCL_COMPILER(MSVC) -# if _CCCL_CTK_BELOW(12, 4) - fprintf(stderr, "Waiving test: conditional nodes are only available since CUDA 12.4.\n"); - return 0; -# else - stackable_ctx ctx; - - // Initialize the time loop - size_t timesteps = 10; - if (argc > 1) - { - timesteps = (size_t) atol(argv[1]); - } - - // No output by default - int output_freq = -1; - if (argc > 2) - { - output_freq = atoi(argv[2]); - } - - // Domain dimensions - const size_t SIZE_X = 100; - const size_t SIZE_Y = 100; - const size_t SIZE_Z = 100; - - // Grid spacing - const double DX = 0.01; - const double DY = 0.01; - const double DZ = 0.01; - - // Define the electric and magnetic fields - auto data_shape = shape_of>(SIZE_X, SIZE_Y, SIZE_Z); - auto lEx = ctx.logical_data(data_shape); - auto lEy = ctx.logical_data(data_shape); - auto lEz = ctx.logical_data(data_shape); - auto lHx = ctx.logical_data(data_shape); - auto lHy = ctx.logical_data(data_shape); - auto lHz = ctx.logical_data(data_shape); - - // Define the permittivity and permeability of the medium - auto lepsilon = ctx.logical_data(data_shape); - auto lmu = ctx.logical_data(data_shape); - - const double EPSILON = 8.85e-12; // Permittivity of free space - const double MU = 1.256e-6; // Permeability of free space - - // CFL condition DT <= min(DX, DY, DZ) * sqrt(epsilon_max * mu_max) - double DT = 0.25 * min(min(DX, DY), DZ) * sqrt(EPSILON * MU); - - // Initialize E - ctx.parallel_for(data_shape, lEx.write(), lEy.write(), lEz.write()) - ->*[] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Ex, auto Ey, auto Ez) { - Ex(i, j, k) = 0.0; - Ey(i, j, k) = 0.0; - Ez(i, j, k) = 0.0; - }; - - // Initialize H - ctx.parallel_for(data_shape, lHx.write(), lHy.write(), lHz.write()) - ->*[] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Hx, auto Hy, auto Hz) { - Hx(i, j, k) = 0.0; - Hy(i, j, k) = 0.0; - Hz(i, j, k) = 0.0; - }; - - // Initialize permittivity and permeability fields - ctx.parallel_for(data_shape, lepsilon.write(), lmu.write()) - ->*[=] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto epsilon, auto mu) { - epsilon(i, j, k) = EPSILON; - mu(i, j, k) = MU; - }; - - // Set the source function at the center of the grid - - const size_t center_x = SIZE_X / 2; - const size_t center_y = SIZE_Y / 2; - const size_t center_z = SIZE_Z / 2; - - /* Index shapes for Electric fields, Magnetic fields, and the indices where there is a source */ - box Es({1ul, SIZE_X - 1}, {1ul, SIZE_Y - 1}, {1ul, SIZE_Z - 1}); - box Hs({0ul, SIZE_X - 1}, {0ul, SIZE_Y - 1}, {0ul, SIZE_Z - 1}); - box source_s({center_x, center_x + 1}, {center_y, center_y + 1}, {center_z, center_z + 1}); - - int iterations_per_graph = (output_freq == -1) ? timesteps : output_freq; - - for (size_t n = 0; n < timesteps / iterations_per_graph; n++) - { - fprintf(stderr, "WHILE BLAAAAA...\n"); - // Counter for while loop iterations - auto counter_shape = shape_of>(); - auto lcounter = ctx.logical_data(counter_shape); - // Initialize counter - ctx.parallel_for(box(1), lcounter.write())->*[=] __device__(size_t, auto counter) { - *counter = iterations_per_graph; - }; - - auto while_guard = ctx.while_graph_scope(); - { - // Update the electric fields - - // Update Ex - ctx.parallel_for(Es, lEx.rw(), lHy.read(), lHz.read(), lepsilon.read()) - ->*[=] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Ex, auto Hy, auto Hz, auto epsilon) { - Ex(i, j, k) = - Ex(i, j, k) - + (DT / (epsilon(i, j, k) * DX)) * (Hz(i, j, k) - Hz(i, j - 1, k) - Hy(i, j, k) + Hy(i, j, k - 1)); - }; - - // Update Ey - ctx.parallel_for(Es, lEy.rw(), lHx.read(), lHz.read(), lepsilon.read()) - ->*[=] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Ey, auto Hx, auto Hz, auto epsilon) { - Ey(i, j, k) = - Ey(i, j, k) - + (DT / (epsilon(i, j, k) * DY)) * (Hx(i, j, k) - Hx(i, j, k - 1) - Hz(i, j, k) + Hz(i - 1, j, k)); - }; - - // Update Ez - ctx.parallel_for(Es, lEz.rw(), lHx.read(), lHy.read(), lepsilon.read()) - ->*[=] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Ez, auto Hx, auto Hy, auto epsilon) { - Ez(i, j, k) = - Ez(i, j, k) - + (DT / (epsilon(i, j, k) * DZ)) * (Hy(i, j, k) - Hy(i - 1, j, k) - Hx(i, j, k) + Hx(i, j - 1, k)); - }; - - // Add the source function at the center of the grid - ctx.parallel_for(source_s, lEz.rw())->*[=] _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Ez) { - Ez(i, j, k) = Ez(i, j, k) + Source(n * DT, i * DX, j * DY, k * DZ); - }; - - // Update the magnetic fields - - // Update Hx - ctx.parallel_for(Hs, lHx.rw(), lEy.read(), lEz.read(), lmu.read()) - ->*[=] - _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Hx, auto Ey, auto Ez, auto mu) { - Hx(i, j, k) = - Hx(i, j, k) - (DT / (mu(i, j, k) * DY)) * (Ez(i, j + 1, k) - Ez(i, j, k) - Ey(i, j, k + 1) + Ey(i, j, k)); - }; - - // Update Hy - ctx.parallel_for(Hs, lHy.rw(), lEx.read(), lEz.read(), lmu.read()) - ->*[=] - _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Hy, auto Ex, auto Ez, auto mu) { - Hy(i, j, k) = - Hy(i, j, k) - (DT / (mu(i, j, k) * DZ)) * (Ex(i, j, k + 1) - Ex(i, j, k) - Ez(i + 1, j, k) + Ez(i, j, k)); - }; - - // Update Hz - ctx.parallel_for(Hs, lHz.rw(), lEx.read(), lEy.read(), lmu.read()) - ->*[=] - _CCCL_DEVICE(size_t i, size_t j, size_t k, auto Hz, auto Ex, auto Ey, auto mu) { - Hz(i, j, k) = - Hz(i, j, k) - (DT / (mu(i, j, k) * DX)) * (Ey(i + 1, j, k) - Ey(i, j, k) - Ex(i, j + 1, k) + Ex(i, j, k)); - }; - - auto handle = while_guard.cond_handle(); - ctx.parallel_for(box(1), lcounter.rw())->*[handle] __device__(size_t, auto counter) { - (*counter)--; - bool should_continue = (*counter > 0); - cudaGraphSetConditional(handle, should_continue); - }; - - } // end of the while pattern - - if (output_freq > 0 && n % output_freq == 0) - { - ctx.host_launch(lEz.read())->*[=](auto Ez) { - // Output the electric field at the center of the grid - fprintf(stderr, "%ld\t%le\n", n, Ez(center_x, center_y, center_z)); - - std::string filename = "Ez" + std::to_string(n) + ".vtk"; - - // Dump a 2D slice of Ez in VTK - write_vtk_2D(filename, Ez, DX, DY, DZ); - }; - } - } - - ctx.finalize(); -# endif // _CCCL_CTK_AT_LEAST(12, 4) -#endif // !_CCCL_COMPILER(MSVC) -} diff --git a/cccl_upstream/cudax/examples/stf/frozen_data_init.cu b/cccl_upstream/cudax/examples/stf/frozen_data_init.cu deleted file mode 100644 index e7a10f61..00000000 --- a/cccl_upstream/cudax/examples/stf/frozen_data_init.cu +++ /dev/null @@ -1,53 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Illustrate how we can use frozen data to initialize constant data - * - */ - -#include - -using namespace cuda::experimental::stf; - -int main() -{ - context ctx; - - /* Create a piece of data that can be use many times without further synchronizations */ - auto buffer = ctx.logical_data(shape_of>(128, 64)).set_symbol("buffer"); - ctx.parallel_for(buffer.shape(), buffer.write())->*[] __device__(size_t i, size_t j, auto b) { - b(i, j) = sin(-1.0 * i) + cos(2.0 * j); - }; - - auto frozen_buffer = ctx.freeze(buffer); - - auto h_buf = frozen_buffer.get(data_place::host()).first; - auto d_buf = frozen_buffer.get(data_place::current_device()).first; - - cuda_safe_call(cudaStreamSynchronize(ctx.fence())); - - auto lX = ctx.logical_data(buffer.shape()).set_symbol("X"); - ctx.parallel_for(lX.shape(), lX.write()).set_symbol("X=buf")->*[d_buf] __device__(size_t i, size_t j, auto x) { - x(i, j) = d_buf(i, j); - }; - - ctx.parallel_for(exec_place::host(), lX.shape(), lX.read()).set_symbol("check buf") - ->*[h_buf](size_t i, size_t j, auto x) { - EXPECT(fabs(x(i, j) - h_buf(i, j)) < 0.0001); - }; - - // Make sure all tasks are done before unfreezing - frozen_buffer.unfreeze(ctx.fence()); - - ctx.finalize(); -} diff --git a/cccl_upstream/cudax/examples/stf/graph_algorithms/degree_centrality.cu b/cccl_upstream/cudax/examples/stf/graph_algorithms/degree_centrality.cu deleted file mode 100644 index 58fc3a61..00000000 --- a/cccl_upstream/cudax/examples/stf/graph_algorithms/degree_centrality.cu +++ /dev/null @@ -1,64 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Computes the Degree Centrality for each vertex within a graph - * - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -/** - * @brief Computes the Degree Centrality for each vertex. - * - * @param idx The index of the vertex for which Degree Centrality is being calculated. - * @param d_offsets Slice containing the offset vector of the CSR representation. - * @return The degree of each vertex. - */ -__device__ int degree_centrality(int idx, slice loffsets) -{ - return loffsets[idx + 1] - loffsets[idx]; -} - -int main() -{ - stream_ctx ctx; - - // row offsets in CSR format - std::vector offsets = {0, 4, 11, 12, 14, 15, 16, 18, 19, 20}; - // edges in CSR format - std::vector nonzeros = {1, 2, 3, 6, 0, 3, 4, 5, 6, 7, 8, 0, 0, 1, 1, 1, 0, 1, 1, 1}; - // output degrees for each vertex - int num_vertices = offsets.size() - 1; - std::vector degrees(num_vertices, 0); - - auto loffsets = ctx.logical_data(&offsets[0], offsets.size()); - auto lnonzeros = ctx.logical_data(&nonzeros[0], nonzeros.size()); - auto ldegrees = ctx.logical_data(°rees[0], degrees.size()); - - ctx.parallel_for(box(num_vertices), loffsets.read(), ldegrees.rw()) - ->*[] __device__(size_t idx, auto loffsets, auto ldegrees) { - ldegrees[idx] = degree_centrality(idx, loffsets); - }; - - ctx.finalize(); - - // for (int i = 0; i < num_vertices; ++i) { - // printf("Vertex %d: Degree Centrality = %d\n", i, degrees[i]); - // } - - return 0; -} diff --git a/cccl_upstream/cudax/examples/stf/graph_algorithms/jaccard.cu b/cccl_upstream/cudax/examples/stf/graph_algorithms/jaccard.cu deleted file mode 100644 index 86a59c2a..00000000 --- a/cccl_upstream/cudax/examples/stf/graph_algorithms/jaccard.cu +++ /dev/null @@ -1,137 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Computes the Jaccard Similarity for each vertex within a graph - * - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -// Performs Binary Search on a given array with start/end bounds and a lookup element -__device__ int binary_search(slice arr, int start, int end, int lookup) -{ - while (start <= end) - { - int mid = start + (end - start) / 2; - if (arr[mid] == lookup) - { - return mid; - } - else if (arr[mid] < lookup) - { - start = mid + 1; - } - else - { - end = mid - 1; - } - } - return -1; -} - -/** - * @brief Computes the intersection size of neighbors of two vertices. - * - * @param loffsets Slice containing the offset vector of the CSR representation. - * @param lnonzeros Slice containing the non-zero elements (neighbors) vector of the CSR representation. - * @param u Index of the first vertex. - * @param v Index of the second vertex. - * @return The number of common neighbors (intersection size) of vertices u and v. - */ -__device__ int calculate_intersection_size(slice loffsets, slice lnonzeros, int u, int v) -{ - int count = 0; - for (int i = loffsets[u]; i < loffsets[u + 1]; i++) - { - if (binary_search(lnonzeros, loffsets[v], loffsets[v + 1] - 1, lnonzeros[i]) != -1) - { - count++; - } - } - return count; -} - -/** - * @brief Computes the union size of neighbors of two vertices. - * - * @param loffsets Slice containing the offset vector of the CSR representation. - * @param lnonzeros Slice containing the non-zero elements (neighbors) vector of the CSR representation. - * @param u Index of the first vertex. - * @param v Index of the second vertex. - * @return The number of unique neighbors (union size) of vertices u and v. - */ -__device__ int calculate_union_size(slice loffsets, slice lnonzeros, int u, int v) -{ - int count = (loffsets[u + 1] - loffsets[u]) + (loffsets[v + 1] - loffsets[v]); - for (int i = loffsets[u]; i < loffsets[u + 1]; i++) - { - if (binary_search(lnonzeros, loffsets[v], loffsets[v + 1] - 1, lnonzeros[i]) != -1) - { - count--; - } - } - return count; -} - -int main() -{ - stream_ctx ctx; - - // row offsets in CSR format - std::vector offsets = {0, 4, 11, 12, 14, 15, 16, 18, 19, 20}; - // edges in CSR format - std::vector nonzeros = {1, 2, 3, 6, 0, 3, 4, 5, 6, 7, 8, 0, 0, 1, 1, 1, 0, 1, 1, 1}; - // output jaccard similarities for each vertex - int num_vertices = offsets.size() - 1; - std::vector jaccard_similarities(num_vertices * num_vertices, 0.0f); - - auto loffsets = ctx.logical_data(&offsets[0], offsets.size()); - auto lnonzeros = ctx.logical_data(&nonzeros[0], nonzeros.size()); - auto ljaccard_similarities = ctx.logical_data(&jaccard_similarities[0], jaccard_similarities.size()); - - ctx.parallel_for(box(num_vertices), loffsets.read(), lnonzeros.read(), ljaccard_similarities.rw()) - ->*[] __device__(size_t idx, auto loffsets, auto lnonzeros, auto ljaccard_similarities) { - for (int j = 0; j < loffsets.size() - 1; j++) - { - if (idx != j) - { - int intersection = calculate_intersection_size(loffsets, lnonzeros, idx, j); - int uni = calculate_union_size(loffsets, lnonzeros, idx, j); - if (uni > 0) - { - ljaccard_similarities[idx * (loffsets.size() - 1) + j] = static_cast(intersection) / uni; - } - } - } - }; - - ctx.finalize(); - - for (int u = 0; u < num_vertices; u++) - { - for (int v = 0; v < num_vertices; v++) - { - if (u != v) - { - printf( - "Jaccard similarity between vertex %d and vertex %d: %f\n", u, v, jaccard_similarities[u * num_vertices + v]); - } - } - } - - return 0; -} diff --git a/cccl_upstream/cudax/examples/stf/graph_algorithms/pagerank.cu b/cccl_upstream/cudax/examples/stf/graph_algorithms/pagerank.cu deleted file mode 100644 index c39cdd68..00000000 --- a/cccl_upstream/cudax/examples/stf/graph_algorithms/pagerank.cu +++ /dev/null @@ -1,121 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Computes the PageRank for vertices within a graph - * - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -/** - * @brief Calculates the PageRank for a given vertex. - * - * @param idx The index of the vertex for which PageRank is being calculated. - * @param loffsets Slice containing the offset vector of the CSR representation. - * @param lnonzeros Slice containing the non-zero elements (neighbors) vector of the CSR representation. - * @param lpage_rank Slice containing current PageRank values for each vertex. - * @param lnew_page_rank Slice containing where new PageRank values will be stored. - * @param init_rank The initial PageRank value to be used in the calculation. - */ -__device__ void calculating_pagerank( - int idx, - const slice& loffsets, - const slice& lnonzeros, - const slice& lpage_rank, - slice& lnew_page_rank, - float init_rank) -{ - float rank_sum = 0.0; - for (int i = loffsets[idx]; i < loffsets[idx + 1]; i++) - { - int neighbor = lnonzeros[i]; - int out_degree = loffsets[neighbor + 1] - loffsets[neighbor]; - rank_sum += lpage_rank[neighbor] / out_degree; - } - lnew_page_rank[idx] = 0.85 * rank_sum + (1.0 - 0.85) * init_rank; -} - -int main() -{ - stream_ctx ctx; - - // row offsets in CSR format - std::vector offsets = {0, 4, 11, 12, 14, 15, 16, 18, 19, 20}; - // edges in CSR format - std::vector nonzeros = {1, 2, 3, 6, 0, 3, 4, 5, 6, 7, 8, 0, 0, 1, 1, 1, 0, 1, 1, 1}; - - int num_vertices = offsets.size() - 1; - float init_rank = 1.0f / num_vertices; - float tolerance = 1e-6f; - int NITER = 100; - - // output pageranks for each vertex - std::vector page_rank(num_vertices, init_rank); - std::vector new_page_rank(num_vertices); - - auto loffsets = ctx.logical_data(&offsets[0], offsets.size()); - auto lnonzeros = ctx.logical_data(&nonzeros[0], nonzeros.size()); - auto lpage_rank = ctx.logical_data(&page_rank[0], page_rank.size()); - auto lnew_page_rank = ctx.logical_data(&new_page_rank[0], new_page_rank.size()); - auto lmax_diff = ctx.logical_data(shape_of>()); - - for (int iter = 0; iter < NITER; ++iter) - { - // Calculate Current Iteration PageRank - ctx.parallel_for( - box(num_vertices), - loffsets.read(), - lnonzeros.read(), - lpage_rank.rw(), - lnew_page_rank.rw(), - lmax_diff.reduce(reducer::maxval{})) - ->*[init_rank] __device__( - size_t idx, auto loffsets, auto lnonzeros, auto lpage_rank, auto lnew_page_rank, auto& max_diff) { - calculating_pagerank(idx, loffsets, lnonzeros, lpage_rank, lnew_page_rank, init_rank); - max_diff = ::std::max(max_diff, lnew_page_rank[idx] - lpage_rank[idx]); - }; - - // Reduce Error and Check for Convergence - bool converged = (ctx.wait(lmax_diff) < tolerance); - if (converged) - { - break; - } - - // Update New PageRank Values - std::swap(lpage_rank, lnew_page_rank); - } - - ctx.finalize(); - - /* CHECKING FOR ANSWER CORRECTNESS */ - // sum of all page ranks should equal 1 - double sum_pageranks = 0.0; - for (int64_t i = 0; i < num_vertices; i++) - { - sum_pageranks += page_rank[i]; - } - printf("Page rank answer is %s.\n", abs(sum_pageranks - 1.0) < 0.001 ? "correct" : "not correct"); - - printf("PageRank Results:\n"); - for (size_t i = 0; i < page_rank.size(); ++i) - { - printf("Vertex %zu: %f\n", i, page_rank[i]); - } - - return 0; -} diff --git a/cccl_upstream/cudax/examples/stf/graph_algorithms/pagerank_batched.cu b/cccl_upstream/cudax/examples/stf/graph_algorithms/pagerank_batched.cu deleted file mode 100644 index 2ca754e8..00000000 --- a/cccl_upstream/cudax/examples/stf/graph_algorithms/pagerank_batched.cu +++ /dev/null @@ -1,203 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Computes the PageRank for vertices within a graph - * - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -#if _CCCL_CTK_AT_LEAST(12, 4) -/** - * @brief Calculates the PageRank for a given vertex. - * - * @param idx The index of the vertex for which PageRank is being calculated. - * @param loffsets Slice containing the offset vector of the CSR representation. - * @param lnonzeros Slice containing the non-zero elements (neighbors) vector of the CSR representation. - * @param lpage_rank Slice containing current PageRank values for each vertex. - * @param lnew_page_rank Slice containing where new PageRank values will be stored. - * @param lpersonalization Slice containing the personalization vector for each vertex. - */ -__device__ void calculating_pagerank( - int idx, - const slice& loffsets, - const slice& lnonzeros, - const slice& lpage_rank, - slice& lnew_page_rank, - const slice& lpersonalization) -{ - float rank_sum = 0.0; - for (int i = loffsets[idx]; i < loffsets[idx + 1]; i++) - { - int neighbor = lnonzeros[i]; - int out_degree = loffsets[neighbor + 1] - loffsets[neighbor]; - rank_sum += lpage_rank[neighbor] / out_degree; - } - lnew_page_rank[idx] = 0.85 * rank_sum + (1.0 - 0.85) * lpersonalization[idx]; -} - -/** - * @brief Computes PageRank using the power iteration method - * - * @param ctx The CUDASTF context - * @param loffsets Logical data for CSR offset vector - * @param lnonzeros Logical data for CSR non-zero elements vector - * @param lpage_rank Logical data for current PageRank values - * @param lpersonalization Logical data for personalization vector - * @param num_vertices Number of vertices in the graph - * @param NITER Maximum number of iterations - * @param tolerance Convergence tolerance - */ -void compute_pagerank( - stackable_ctx& ctx, - stackable_logical_data>& loffsets, - stackable_logical_data>& lnonzeros, - stackable_logical_data>& lpage_rank, - stackable_logical_data>& lpersonalization, - int num_vertices, - int NITER, - float tolerance) -{ - // Create local temporary buffer and convergence tracking - auto lnew_page_rank = ctx.logical_data(lpage_rank.shape()); - auto lmax_diff = ctx.logical_data(shape_of>()); - auto liter = ctx.logical_data(shape_of>()); - - // Initialize iteration counter - ctx.parallel_for(box(1), liter.write())->*[] __device__(size_t, auto iter) { - *iter = 0; - }; - - { - auto while_guard = ctx.while_graph_scope(); - - // Calculate Current Iteration PageRank - ctx.parallel_for( - box(num_vertices), - loffsets.read(), - lnonzeros.read(), - lpage_rank.rw(), - lnew_page_rank.write(), - lpersonalization.read(), - lmax_diff.reduce(reducer::maxval{})) - ->* - [] __device__( - size_t idx, - auto loffsets, - auto lnonzeros, - auto lpage_rank, - auto lnew_page_rank, - auto lpersonalization, - auto& max_diff) { - calculating_pagerank(idx, loffsets, lnonzeros, lpage_rank, lnew_page_rank, lpersonalization); - max_diff = ::std::max(max_diff, lnew_page_rank[idx] - lpage_rank[idx]); - }; - - // Update PageRank Values - ctx.parallel_for(lpage_rank.shape(), lpage_rank.write(), lnew_page_rank.read()) - ->*[] __device__(size_t i, auto page_rank, auto new_page_rank) { - page_rank(i) = new_page_rank(i); - }; - - while_guard.update_cond(lmax_diff.read(), liter.rw())->*[NITER, tolerance] __device__(auto max_diff, auto iter) { - bool converged = (*max_diff < tolerance); - bool max_reached = ((*iter)++ >= NITER); // Maximum iteration limit - return !converged && !max_reached; // Continue if not converged and under limit - }; - } -} -#endif // _CCCL_CTK_AT_LEAST(12, 4) - -int main() -{ -#if _CCCL_CTK_BELOW(12, 4) - fprintf(stderr, "Waiving example: while_graph_scope is only available since CUDA 12.4.\n"); - return 0; -#else - stackable_ctx ctx; - - // row offsets in CSR format - std::vector offsets = {0, 4, 11, 12, 14, 15, 16, 18, 19, 20}; - // edges in CSR format - std::vector nonzeros = {1, 2, 3, 6, 0, 3, 4, 5, 6, 7, 8, 0, 0, 1, 1, 1, 0, 1, 1, 1}; - - int num_vertices = offsets.size() - 1; - float init_rank = 1.0f / num_vertices; - float tolerance = 1e-6f; - int NITER = 100; - int num_personalization = 4; - - ::std::vector>> lpage_rank_slices; - for (int i = 0; i < num_personalization; i++) - { - lpage_rank_slices.push_back(ctx.logical_data(shape_of>(num_vertices))); - } - - auto loffsets = ctx.logical_data(&offsets[0], offsets.size()); - auto lnonzeros = ctx.logical_data(&nonzeros[0], nonzeros.size()); - - loffsets.set_read_only(); - lnonzeros.set_read_only(); - - { - auto scope = ctx.graph_scope(); - for (int p = 0; p < num_personalization; p++) - { - // Initialize PageRank values to uniform distribution - ctx.parallel_for(lpage_rank_slices[p].shape(), lpage_rank_slices[p].write()) - ->*[init_rank] __device__(size_t i, auto page_rank) { - page_rank(i) = init_rank; - }; - - // Create personalization vector (uniform for this example) - auto lpersonalization = ctx.logical_data(shape_of>(num_vertices)); - ctx.parallel_for(lpersonalization.shape(), lpersonalization.write()) - ->*[init_rank] __device__(size_t i, auto lpersonalization) { - lpersonalization(i) = init_rank; - }; - - compute_pagerank(ctx, loffsets, lnonzeros, lpage_rank_slices[p], lpersonalization, num_vertices, NITER, tolerance); - } - } - - for (int p = 0; p < num_personalization; p++) - { - ctx.host_launch(lpage_rank_slices[p].read())->*[p, num_vertices] __host__(slice page_rank) { - double sum_pageranks = 0.0; - for (int64_t i = 0; i < num_vertices; i++) - { - sum_pageranks += page_rank[i]; - } - printf("Page rank answer for personalization %d is %s.\n", - p, - abs(sum_pageranks - 1.0) < 0.001 ? "correct" : "not correct"); - - // Print first few results for verification - printf("Personalization %d - First 5 vertices: ", p); - for (size_t i = 0; i < std::min(5UL, page_rank.size()); ++i) - { - printf("%.6f ", page_rank[i]); - } - printf("\n"); - }; - } - - ctx.finalize(); - - return 0; -#endif // !_CCCL_CTK_BELOW(12, 4) -} diff --git a/cccl_upstream/cudax/examples/stf/graph_algorithms/pagerank_while.cu b/cccl_upstream/cudax/examples/stf/graph_algorithms/pagerank_while.cu deleted file mode 100644 index 6e6299c1..00000000 --- a/cccl_upstream/cudax/examples/stf/graph_algorithms/pagerank_while.cu +++ /dev/null @@ -1,135 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Computes the PageRank for vertices within a graph - * - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -/** - * @brief Calculates the PageRank for a given vertex. - * - * @param idx The index of the vertex for which PageRank is being calculated. - * @param loffsets Slice containing the offset vector of the CSR representation. - * @param lnonzeros Slice containing the non-zero elements (neighbors) vector of the CSR representation. - * @param lpage_rank Slice containing current PageRank values for each vertex. - * @param lnew_page_rank Slice containing where new PageRank values will be stored. - * @param init_rank The initial PageRank value to be used in the calculation. - */ -__device__ void calculating_pagerank( - int idx, - const slice& loffsets, - const slice& lnonzeros, - const slice& lpage_rank, - slice& lnew_page_rank, - float init_rank) -{ - float rank_sum = 0.0; - for (int i = loffsets[idx]; i < loffsets[idx + 1]; i++) - { - int neighbor = lnonzeros[i]; - int out_degree = loffsets[neighbor + 1] - loffsets[neighbor]; - rank_sum += lpage_rank[neighbor] / out_degree; - } - lnew_page_rank[idx] = 0.85 * rank_sum + (1.0 - 0.85) * init_rank; -} - -int main() -{ -#if _CCCL_CTK_BELOW(12, 4) - fprintf(stderr, "Waiving example: while_graph_scope is only available since CUDA 12.4.\n"); - return 0; -#else - stackable_ctx ctx; - - // row offsets in CSR format - std::vector offsets = {0, 4, 11, 12, 14, 15, 16, 18, 19, 20}; - // edges in CSR format - std::vector nonzeros = {1, 2, 3, 6, 0, 3, 4, 5, 6, 7, 8, 0, 0, 1, 1, 1, 0, 1, 1, 1}; - - int num_vertices = offsets.size() - 1; - float init_rank = 1.0f / num_vertices; - float tolerance = 1e-6f; - int NITER = 100; - - // output pageranks for each vertex - std::vector page_rank(num_vertices, init_rank); - std::vector new_page_rank(num_vertices); - - auto loffsets = ctx.logical_data(&offsets[0], offsets.size()); - auto lnonzeros = ctx.logical_data(&nonzeros[0], nonzeros.size()); - auto lpage_rank = ctx.logical_data(&page_rank[0], page_rank.size()); - auto lnew_page_rank = ctx.logical_data(&new_page_rank[0], new_page_rank.size()); - auto lmax_diff = ctx.logical_data(shape_of>()); - auto liter = ctx.logical_data(shape_of>()); - - // Initialize iteration counter - ctx.parallel_for(box(1), liter.write())->*[] __device__(size_t, auto iter) { - *iter = 0; - }; - - { - auto while_guard = ctx.while_graph_scope(); - - // Calculate Current Iteration PageRank - ctx.parallel_for( - box(num_vertices), - loffsets.read(), - lnonzeros.read(), - lpage_rank.rw(), - lnew_page_rank.rw(), - lmax_diff.reduce(reducer::maxval{})) - ->*[init_rank] __device__( - size_t idx, auto loffsets, auto lnonzeros, auto lpage_rank, auto lnew_page_rank, auto& max_diff) { - calculating_pagerank(idx, loffsets, lnonzeros, lpage_rank, lnew_page_rank, init_rank); - max_diff = ::std::max(max_diff, lnew_page_rank[idx] - lpage_rank[idx]); - }; - - // Update PageRank Values - ctx.parallel_for(lpage_rank.shape(), lpage_rank.write(), lnew_page_rank.read()) - ->*[] __device__(size_t i, auto page_rank, auto new_page_rank) { - page_rank(i) = new_page_rank(i); - }; - - while_guard.update_cond(lmax_diff.read(), liter.rw())->*[NITER, tolerance] __device__(auto max_diff, auto iter) { - bool converged = (*max_diff < tolerance); - bool max_reached = ((*iter)++ >= NITER); // Maximum iteration limit - return !converged && !max_reached; // Continue if not converged and under limit - }; - } - - ctx.finalize(); - - /* CHECKING FOR ANSWER CORRECTNESS */ - // sum of all page ranks should equal 1 - double sum_pageranks = 0.0; - for (int64_t i = 0; i < num_vertices; i++) - { - sum_pageranks += page_rank[i]; - } - printf("Page rank answer is %s.\n", abs(sum_pageranks - 1.0) < 0.001 ? "correct" : "not correct"); - - printf("PageRank Results:\n"); - for (size_t i = 0; i < page_rank.size(); ++i) - { - printf("Vertex %zu: %f\n", i, page_rank[i]); - } - - return 0; -#endif // !_CCCL_CTK_BELOW(12, 4) -} diff --git a/cccl_upstream/cudax/examples/stf/graph_algorithms/tricount.cu b/cccl_upstream/cudax/examples/stf/graph_algorithms/tricount.cu deleted file mode 100644 index b79c351c..00000000 --- a/cccl_upstream/cudax/examples/stf/graph_algorithms/tricount.cu +++ /dev/null @@ -1,100 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Computes the total number of triangles within a graph - * - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -// Performs Binary Search on a given array with start/end bounds and a lookup element -__device__ int binary_search(slice arr, int start, int end, int lookup) -{ - while (start <= end) - { - int mid = start + (end - start) / 2; - if (arr[mid] == lookup) - { - return mid; - } - else if (arr[mid] < lookup) - { - start = mid + 1; - } - else - { - end = mid - 1; - } - } - return -1; -} - -/** - * @brief Computes the Triangle Counting for each vertex. - * - * @param idx The index of the vertex for which Triangle Counting is being calculated. - * @param loffsets Slice containing the offset vector of the CSR representation. - * @param lnonzeros Slice containing the non-zero elements (neighbors) vector of the CSR representation. - * @return The local triangle count for the vertex. - */ -__device__ unsigned long long int triangle_count(int idx, slice loffsets, slice lnonzeros) -{ - int lcount = 0; - for (int i = loffsets[idx]; i < loffsets[idx + 1]; i++) - { - int v = lnonzeros[i]; - for (int j = loffsets[idx]; j < loffsets[idx + 1]; j++) - { - int w = lnonzeros[j]; - if (binary_search(lnonzeros, loffsets[v], loffsets[v + 1] - 1, w) != -1) - { - lcount++; - } - } - } - return lcount; -} - -int main() -{ - stream_ctx ctx; - - // row offsets in CSR format - std::vector offsets = {0, 0, 1, 2, 4, 5, 6, 8, 9, 10}; - // edges in CSR format - std::vector nonzeros = {0, 0, 0, 1, 1, 1, 0, 1, 1, 1}; - - int num_vertices = offsets.size() - 1; - - auto loffsets = ctx.logical_data(&offsets[0], offsets.size()); - auto lnonzeros = ctx.logical_data(&nonzeros[0], nonzeros.size()); - auto ltotal_count = ctx.logical_data(shape_of>()); - - ctx.parallel_for( - box(num_vertices), loffsets.read(), lnonzeros.read(), ltotal_count.reduce(reducer::sum{})) - ->*[] __device__(size_t idx, auto loffsets, auto lnonzeros, auto& total_count) { - total_count += triangle_count(idx, loffsets, lnonzeros); - }; - - auto total_count = ctx.wait(ltotal_count); - - ctx.finalize(); - - printf("Number of triangles: %lld\n", total_count); - - return 0; -} diff --git a/cccl_upstream/cudax/examples/stf/graph_scope.cu b/cccl_upstream/cudax/examples/stf/graph_scope.cu deleted file mode 100644 index b49db56a..00000000 --- a/cccl_upstream/cudax/examples/stf/graph_scope.cu +++ /dev/null @@ -1,105 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Demonstration of graph_scope RAII usage styles - * - * This example shows different ways to use stackable_ctx::graph_scope_guard - * for automatic push/pop management in nested contexts. - */ - -#include - -using namespace cuda::experimental::stf; - -int main() -{ - stackable_ctx ctx; - - int data[10] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - auto lA = ctx.logical_data(data); - - // Style 1: Direct constructor (like std::lock_guard) - // This is the most idiomatic C++ style - { - stackable_ctx::graph_scope_guard scope{ctx}; // Direct constructor - push() called - - auto temp = ctx.logical_data(lA.shape()); - ctx.parallel_for(temp.shape(), temp.write(), lA.read())->*[] __device__(size_t i, auto temp, auto a) { - temp(i) = a(i) * 2; - }; - - ctx.parallel_for(lA.shape(), lA.write(), temp.read())->*[] __device__(size_t i, auto a, auto temp) { - a(i) = temp(i); - }; - - // pop() called automatically when scope goes out of scope - } - - // Style 2: Factory method (convenience) - // Useful when you prefer auto type deduction - { - auto scope = ctx.graph_scope(); // Factory method - push() called - - ctx.parallel_for(lA.shape(), lA.rw())->*[] __device__(size_t i, auto a) { - a(i) += 1; - }; - - // pop() called automatically - } - - // Style 3: Direct constructor with explicit type alias - // Useful for readability in complex scenarios - { - using scope_t = stackable_ctx::graph_scope_guard; - scope_t scope{ctx}; // Explicit type - push() called - - ctx.parallel_for(lA.shape(), lA.rw())->*[] __device__(size_t i, auto a) { - a(i) *= 3; - }; - - // pop() called automatically - } - - // Style 4: Iterative pattern (like in stackable2.cu) - // Demonstrates repeated nested contexts - for (int iter = 0; iter < 3; iter++) - { - stackable_ctx::graph_scope_guard iteration{ctx}; // New scope each iteration - - auto temp = ctx.logical_data(lA.shape()); - - // tmp = a - ctx.parallel_for(temp.shape(), temp.write(), lA.read())->*[] __device__(size_t i, auto temp, auto a) { - temp(i) = a(i); - }; - - // a++ - ctx.parallel_for(lA.shape(), lA.rw())->*[] __device__(size_t i, auto a) { - a(i) += 1; - }; - - // tmp *= 2 - ctx.parallel_for(temp.shape(), temp.rw())->*[] __device__(size_t i, auto temp) { - temp(i) *= 2; - }; - - // a += tmp - ctx.parallel_for(lA.shape(), temp.read(), lA.rw())->*[] __device__(size_t i, auto temp, auto a) { - a(i) += temp(i); - }; - - // pop() called automatically at end of iteration - } - - ctx.finalize(); - return 0; -} diff --git a/cccl_upstream/cudax/examples/stf/heat.cu b/cccl_upstream/cudax/examples/stf/heat.cu deleted file mode 100644 index b69dfc4c..00000000 --- a/cccl_upstream/cudax/examples/stf/heat.cu +++ /dev/null @@ -1,126 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief An example solving heat equation with finite differences using the - * parallel_for construct. - * - * A multi-gpu version is shown in the heat_mgpu.cu example. - * - * This example also illustrate how to annotate resources with set_symbol - */ - -#include - -using namespace cuda::experimental::stf; - -void dump_iter(slice sUn, int iter) -{ - /* Create a binary file in the PPM format */ - char name[64]; - snprintf(name, 64, "heat_%06d.ppm", iter); - FILE* f = fopen(name, "wb"); - fprintf(f, "P6\n%zu %zu\n255\n", sUn.extent(0), sUn.extent(1)); - for (size_t j = 0; j < sUn.extent(1); j++) - { - for (size_t i = 0; i < sUn.extent(0); i++) - { - int v = (int) (255.0 * sUn(i, j) / 100.0); - // we assume values between 0.0 and 100.0 : max value is in red, - // min is in blue - unsigned char color[3]; - color[0] = static_cast(v); /* red */ - color[1] = static_cast(0); /* green */ - color[2] = static_cast(255 - v); /* blue */ - fwrite(color, 1, 3, f); - } - } - fclose(f); -} - -int main() -{ - context ctx; - - const size_t N = 800; - - auto lU = ctx.logical_data(shape_of>(N, N)); - auto lU1 = ctx.logical_data(lU.shape()); - - // Initialize the Un field with boundary conditions, and a disk at a lower - // temperature in the middle. - ctx.parallel_for(lU.shape(), lU.write())->*[=] _CCCL_DEVICE(size_t i, size_t j, auto U) { - double rad = U.extent(0) / 8.0; - double dx = (double) i - U.extent(0) / 2; - double dy = (double) j - U.extent(1) / 2; - - U(i, j) = (dx * dx + dy * dy < rad * rad) ? 100.0 : 0.0; - - /* Set up boundary conditions */ - if (j == 0.0) - { - U(i, j) = 100.0; - } - if (j == U.extent(1) - 1) - { - U(i, j) = 0.0; - } - if (i == 0.0) - { - U(i, j) = 0.0; - } - if (i == U.extent(0) - 1) - { - U(i, j) = 0.0; - } - }; - - // diffusion constant - double a = 0.5; - - double dx = 0.1; - double dy = 0.1; - double dx2 = dx * dx; - double dy2 = dy * dy; - - // time step - double dt = dx2 * dy2 / (2.0 * a * (dx2 + dy2)); - - double c = a * dt; - - int nsteps = 1000; - int image_freq = -1; - - for (int iter = 0; iter < nsteps; iter++) - { - if (image_freq > 0 && iter % image_freq == 0) - { - // Dump Un in a PPM file - ctx.host_launch(lU.read())->*[=](auto U) { - dump_iter(U, iter); - }; - } - - // Update Un using Un1 value with a finite difference scheme - ctx.parallel_for(inner<1>(lU.shape()), lU.read(), lU1.write()) - ->*[=] - _CCCL_DEVICE(size_t i, size_t j, auto U, auto U1) { - U1(i, j) = - U(i, j) - + c * ((U(i - 1, j) - 2 * U(i, j) + U(i + 1, j)) / dx2 + (U(i, j - 1) - 2 * U(i, j) + U(i, j + 1)) / dy2); - }; - - std::swap(lU, lU1); - } - - ctx.finalize(); -} diff --git a/cccl_upstream/cudax/examples/stf/heat_mgpu.cu b/cccl_upstream/cudax/examples/stf/heat_mgpu.cu deleted file mode 100644 index 90dba544..00000000 --- a/cccl_upstream/cudax/examples/stf/heat_mgpu.cu +++ /dev/null @@ -1,163 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief An example solving heat equation with finite differences on multiple devices - * - * This example also illustrate how to annotate resources with set_symbol - */ - -#include - -using namespace cuda::experimental::stf; - -void dump_iter(slice sUn, int iter) -{ - /* Create a binary file in the PPM format */ - char name[64]; - snprintf(name, 64, "heat_%06d.ppm", iter); - FILE* f = fopen(name, "wb"); - fprintf(f, "P6\n%zu %zu\n255\n", sUn.extent(0), sUn.extent(1)); - for (size_t j = 0; j < sUn.extent(1); j++) - { - for (size_t i = 0; i < sUn.extent(0); i++) - { - int v = (int) (255.0 * sUn(i, j) / 100.0); - // we assume values between 0.0 and 100.0 : max value is in red, - // min is in blue - unsigned char color[3]; - color[0] = static_cast(v); /* red */ - color[1] = static_cast(0); /* green */ - color[2] = static_cast(255 - v); /* blue */ - fwrite(color, 1, 3, f); - } - } - fclose(f); -} - -int main(int argc, char** argv) -{ - context ctx; - - size_t N = 1000; - int nsteps = 100; - int image_freq = -1; - - if (argc > 1) - { - N = atol(argv[1]); - } - - if (argc > 2) - { - nsteps = atoi(argv[2]); - } - - if (argc > 3) - { - image_freq = atoi(argv[3]); - } - - if (argc > 4) - { - int use_graphs = atoi(argv[4]); - if (use_graphs != 0) - { - ctx = graph_ctx(); - } - } - - auto lU = ctx.logical_data(shape_of>(N, N)); - auto lU1 = ctx.logical_data(lU.shape()); - - lU.set_symbol("U"); - lU1.set_symbol("U1"); - - auto all_devs = exec_place::all_devices(); - - // Initialize the Un field with boundary conditions, and a disk at a lower - // temperature in the middle. - ctx.parallel_for(blocked_partition(), all_devs, lU.shape(), lU.write()).set_symbol("init")->* - [=] _CCCL_DEVICE(size_t i, size_t j, auto U) { - double rad = U.extent(0) / 8.0; - double dx = (double) i - U.extent(0) / 2; - double dy = (double) j - U.extent(1) / 2; - - U(i, j) = (dx * dx + dy * dy < rad * rad) ? 100.0 : 0.0; - - /* Set up boundary conditions */ - if (j == 0.0) - { - U(i, j) = 100.0; - } - if (j == U.extent(1) - 1) - { - U(i, j) = 0.0; - } - if (i == 0.0) - { - U(i, j) = 0.0; - } - if (i == U.extent(0) - 1) - { - U(i, j) = 0.0; - } - }; - - // diffusion constant - double a = 0.5; - - double dx = 0.1; - double dy = 0.1; - double dx2 = dx * dx; - double dy2 = dy * dy; - - // time step - double dt = dx2 * dy2 / (2.0 * a * (dx2 + dy2)); - - double c = a * dt; - - cudaEvent_t start, stop; - - cuda_safe_call(cudaEventCreate(&start)); - cuda_safe_call(cudaEventCreate(&stop)); - - cuda_safe_call(cudaEventRecord(start, ctx.fence())); - - ctx.repeat(nsteps)->*[&](context ctx, size_t iter) { - if (image_freq > 0 && iter % image_freq == 0) - { - // Dump Un in a PPM file - ctx.host_launch(lU.read()).set_symbol("dump")->*[=](auto U) { - dump_iter(U, static_cast(iter)); - }; - } - - // Update Un using Un1 value with a finite difference scheme - ctx.parallel_for(blocked_partition(), all_devs, inner<1>(lU.shape()), lU.read(), lU1.write()).set_symbol("step")->* - [=] _CCCL_DEVICE(size_t i, size_t j, auto U, auto U1) { - U1(i, j) = - U(i, j) - + c * ((U(i - 1, j) - 2 * U(i, j) + U(i + 1, j)) / dx2 + (U(i, j - 1) - 2 * U(i, j) + U(i, j + 1)) / dy2); - }; - - std::swap(lU, lU1); - }; - - cuda_safe_call(cudaEventRecord(stop, ctx.fence())); - - ctx.finalize(); - - float elapsedTime; - cudaEventElapsedTime(&elapsedTime, start, stop); - printf("Elapsed time: %f ms\n", elapsedTime); -} diff --git a/cccl_upstream/cudax/examples/stf/jacobi.cu b/cccl_upstream/cudax/examples/stf/jacobi.cu deleted file mode 100644 index 3a60515c..00000000 --- a/cccl_upstream/cudax/examples/stf/jacobi.cu +++ /dev/null @@ -1,153 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Jacobi method with launch - * - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -/* Implement atomicMax with a compare and swap */ -_CCCL_DEVICE double atomicMax(double* address, double val) -{ - unsigned long long int* address_as_ull = (unsigned long long int*) address; - unsigned long long int old = *address_as_ull, assumed; - - do - { - assumed = old; - old = atomicCAS(address_as_ull, assumed, __double_as_longlong(fmax(val, __longlong_as_double(assumed)))); - - // Note: uses integer comparison to avoid hang in case of NaN (since NaN != NaN) - } while (assumed != old); - - return __longlong_as_double(old); -} - -template -_CCCL_DEVICE double reduce_max(thread_hierarchy_t& t, double local_max) -{ - auto ti = t.inner(); - slice error = t.template storage(0); - - error(0) = 0.0; - t.sync(); - - // Note we do not use t.static_width(1) because t is a runtime variable so it - // cannot be used directly to statically evaluate the size. - __shared__ double block_max[thread_hierarchy_t::static_width(1)]; - block_max[ti.rank()] = local_max; - for (size_t s = ti.size() / 2; s > 0; s /= 2) - { - if (ti.rank() < s) - { - block_max[ti.rank()] = fmax(block_max[ti.rank() + s], block_max[ti.rank()]); - } - ti.sync(); - } - - if (ti.rank() == 0) - { - atomicMax(&error(0), block_max[0]); - } - t.sync(); - - return error(0); -} - -int main(int argc, char** argv) -{ - context ctx; - - size_t n = 4096; - size_t m = 4096; - size_t iter_max = 100; - double tol = 0.0000001; - - if (argc > 2) - { - n = atol(argv[1]); - m = atol(argv[2]); - } - - if (argc > 3) - { - iter_max = atoi(argv[3]); - } - - if (argc > 4) - { - tol = atof(argv[4]); - } - - auto lA = ctx.logical_data(shape_of>(m, n)); - auto lAnew = ctx.logical_data(lA.shape()); - - auto all_devs = exec_place::all_devices(); - - ctx.parallel_for(blocked_partition(), all_devs, lA.shape(), lA.write(), lAnew.write()).set_symbol("init")->* - [=] _CCCL_DEVICE(size_t i, size_t j, auto A, auto Anew) { - A(i, j) = (i == j) ? 10.0 : -1.0; - }; - - cudaEvent_t start, stop; - - cuda_safe_call(cudaEventCreate(&start)); - cuda_safe_call(cudaEventCreate(&stop)); - - cuda_safe_call(cudaEventRecord(start, ctx.fence())); - - auto spec = con(con<64>(), mem(sizeof(double))); - - ctx.launch(spec, all_devs, lA.rw(), lAnew.write())->*[iter_max, tol, n, m] _CCCL_DEVICE(auto t, auto A, auto Anew) { - auto ti = t.inner(); - for (size_t iter = 0; iter < iter_max; iter++) - { - // thread-local maximum error - double local_error = 0.0; - - for (auto [i, j] : t.apply_partition(inner<1>(shape(A)))) - { - Anew(i, j) = 0.25 * (A(i - 1, j) + A(i + 1, j) + A(i, j - 1) + A(i, j + 1)); - - local_error = fmax(local_error, fabs(A(i, j) - Anew(i, j))); - } - - // compute the overall maximum error - double error = reduce_max(t, local_error); - - /* Fill A with the new values */ - for (auto [i, j] : t.apply_partition(shape(A))) - { - A(i, j) = Anew(i, j); - } - - if (iter % 25 == 0 && t.rank() == 0) - { - printf("iter %zu : error %e (tol %e)\n", iter, error, tol); - } - } - }; - - cuda_safe_call(cudaEventRecord(stop, ctx.fence())); - - ctx.finalize(); - - float elapsedTime; - cudaEventElapsedTime(&elapsedTime, start, stop); - printf("Elapsed time: %f ms\n", elapsedTime); -} diff --git a/cccl_upstream/cudax/examples/stf/jacobi_pfor.cu b/cccl_upstream/cudax/examples/stf/jacobi_pfor.cu deleted file mode 100644 index 6421b742..00000000 --- a/cccl_upstream/cudax/examples/stf/jacobi_pfor.cu +++ /dev/null @@ -1,95 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Jacobi method with parallel_for - * - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -int main(int argc, char** argv) -{ - context ctx; - - size_t n = 4096; - size_t m = 4096; - double tol = 0.5; - size_t iter_max = 1000; - - if (argc > 2) - { - n = atol(argv[1]); - m = atol(argv[2]); - } - - if (argc > 3) - { - tol = atof(argv[3]); - } - - if (argc > 4) - { - iter_max = atoi(argv[4]); - } - - auto lA = ctx.logical_data(shape_of>(m, n)); - auto lAnew = ctx.logical_data(lA.shape()); - - ctx.parallel_for(lA.shape(), lA.write(), lAnew.write()).set_symbol("init")->* - [=] __device__(size_t i, size_t j, auto A, auto Anew) { - A(i, j) = (i == j) ? 10.0 : -1.0; - Anew(i, j) = A(i, j); - }; - - cudaEvent_t start, stop; - - cuda_safe_call(cudaEventCreate(&start)); - cuda_safe_call(cudaEventCreate(&stop)); - - cuda_safe_call(cudaEventRecord(start, ctx.fence())); - - auto lresidual = ctx.logical_data(shape_of>()); - - size_t iter = 0; - do - { - ctx.parallel_for(inner<1>(lA.shape()), lA.read(), lAnew.rw(), lresidual.reduce(reducer::maxval{})) - ->*[] __device__(size_t i, size_t j, auto A, auto Anew, auto& residual) { - Anew(i, j) = 0.25 * (A(i - 1, j) + A(i + 1, j) + A(i, j - 1) + A(i, j + 1)); - residual = ::std::max(residual, fabs(A(i, j) - Anew(i, j))); - }; - - ctx.parallel_for(inner<1>(lA.shape()), lA.rw(), lAnew.read())->*[] __device__(size_t i, size_t j, auto A, auto Anew) { - A(i, j) = Anew(i, j); - }; - - iter++; - - } while (ctx.wait(lresidual) > tol && iter < iter_max); - - cuda_safe_call(cudaEventRecord(stop, ctx.fence())); - - double final_residual = ctx.wait(lresidual); - - printf("Converged after %ld iterations, residual = %lf\n", iter, final_residual); - - ctx.finalize(); - - float elapsedTime; - cudaEventElapsedTime(&elapsedTime, start, stop); - printf("Elapsed time: %f ms\n", elapsedTime); -} diff --git a/cccl_upstream/cudax/examples/stf/jacobi_stackable.cu b/cccl_upstream/cudax/examples/stf/jacobi_stackable.cu deleted file mode 100644 index 5b182e31..00000000 --- a/cccl_upstream/cudax/examples/stf/jacobi_stackable.cu +++ /dev/null @@ -1,97 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Jacobi method with parallel_for and graphs - * - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) -{ -#if _CCCL_CTK_BELOW(12, 4) - fprintf(stderr, "Waiving test: conditional nodes are only available since CUDA 12.4.\n"); - return 0; -#else - stackable_ctx ctx; - - size_t n = 4096; - size_t m = 4096; - double tol = 0.1; - - if (argc > 2) - { - n = atol(argv[1]); - m = atol(argv[2]); - } - - if (argc > 3) - { - tol = atof(argv[3]); - } - - auto lA = ctx.logical_data(shape_of>(m, n)); - auto lAnew = ctx.logical_data(lA.shape()); - - ctx.parallel_for(lA.shape(), lA.write(), lAnew.write()).set_symbol("init")->* - [=] __device__(size_t i, size_t j, auto A, auto Anew) { - A(i, j) = (i == j) ? 1.0 : -1.0; - }; - - cudaEvent_t start, stop; - - cuda_safe_call(cudaEventCreate(&start)); - cuda_safe_call(cudaEventCreate(&stop)); - - cuda_safe_call(cudaEventRecord(start, ctx.fence())); - - auto lconverged = ctx.logical_data(shape_of>()); - - size_t iter = 0; - - // Creating a conditional handle but not using it in a conditional node can make the graph instantiation fail. - cudaGraphConditionalHandle handle; - ctx.push_while(&handle, 1, cudaGraphCondAssignDefault); - - ctx.parallel_for(inner<1>(lA.shape()), lA.read(), lAnew.write(), lconverged.reduce(reducer::logical_and{})) - ->*[tol] __device__(size_t i, size_t j, auto A, auto Anew, auto& converged) { - Anew(i, j) = 0.25 * (A(i - 1, j) + A(i + 1, j) + A(i, j - 1) + A(i, j + 1)); - double error = fabs(A(i, j) - Anew(i, j)); - converged = converged && (error < tol); - }; - - ctx.parallel_for(inner<1>(lA.shape()), lA.rw(), lAnew.read())->*[] __device__(size_t i, size_t j, auto A, auto Anew) { - A(i, j) = Anew(i, j); - }; - - ctx.parallel_for(box(1), lconverged.read())->*[handle] __device__(size_t, auto converged) { - cudaGraphSetConditional(handle, !*converged); - }; - - ctx.pop(); - - fprintf(stderr, "ITER %zu: converged\n", iter++); - - cuda_safe_call(cudaEventRecord(stop, ctx.fence())); - - ctx.finalize(); - - float elapsedTime; - cudaEventElapsedTime(&elapsedTime, start, stop); - printf("Elapsed time: %f ms\n", elapsedTime); -#endif -} diff --git a/cccl_upstream/cudax/examples/stf/jacobi_stackable_raii.cu b/cccl_upstream/cudax/examples/stf/jacobi_stackable_raii.cu deleted file mode 100644 index 5b6b21e9..00000000 --- a/cccl_upstream/cudax/examples/stf/jacobi_stackable_raii.cu +++ /dev/null @@ -1,104 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Jacobi method with a while scope guard and explicit management of the conditional handle - * - */ - -#include - -#include - -#include "cuda/experimental/__stf/stackable/stackable_ctx.cuh" - -using namespace cuda::experimental::stf; - -int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) -{ -#if _CCCL_CTK_BELOW(12, 4) - fprintf(stderr, "Waiving test: conditional nodes are only available since CUDA 12.4.\n"); - return 0; -#else - stackable_ctx ctx; - - size_t n = 4096; - size_t m = 4096; - double tol = 0.1; - - if (argc > 2) - { - n = atol(argv[1]); - m = atol(argv[2]); - } - - if (argc > 3) - { - tol = atof(argv[3]); - } - - auto lA = ctx.logical_data(shape_of>(m, n)); - auto lAnew = ctx.logical_data(lA.shape()); - - ctx.parallel_for(lA.shape(), lA.write(), lAnew.write()).set_symbol("init")->* - [=] __device__(size_t i, size_t j, auto A, auto Anew) { - A(i, j) = (i == j) ? 1.0 : -1.0; - }; - - cudaEvent_t start, stop; - - cuda_safe_call(cudaEventCreate(&start)); - cuda_safe_call(cudaEventCreate(&stop)); - - cuda_safe_call(cudaEventRecord(start, ctx.fence())); - - size_t iter = 0; - - auto lresidual = ctx.logical_data(shape_of>()); - - { - auto while_guard = ctx.while_graph_scope(); - - ctx.parallel_for(inner<1>(lA.shape()), lA.read(), lAnew.write(), lresidual.reduce(reducer::maxval{})) - ->*[] __device__(size_t i, size_t j, auto A, auto Anew, auto& residual) { - Anew(i, j) = 0.25 * (A(i - 1, j) + A(i + 1, j) + A(i, j - 1) + A(i, j + 1)); - double error = fabs(A(i, j) - Anew(i, j)); - residual = error; - }; - - ctx.parallel_for(inner<1>(lA.shape()), lA.rw(), lAnew.read())->*[] __device__(size_t i, size_t j, auto A, auto Anew) { - A(i, j) = Anew(i, j); - }; - - auto handle = while_guard.cond_handle(); - ctx.parallel_for(box(1), lresidual.read())->*[handle, tol] __device__(size_t, auto residual) { - bool converged = (*residual < tol); - cudaGraphSetConditional(handle, !converged); - }; - } - - // Store final residual for verification - double final_residual = ctx.wait(lresidual); - - fprintf(stderr, "ITER %zu: converged residual %e\n", iter++, final_residual); - - cuda_safe_call(cudaEventRecord(stop, ctx.fence())); - - ctx.finalize(); - - EXPECT(final_residual <= tol); // Algorithm should have converged within tolerance - - float elapsedTime; - cudaEventElapsedTime(&elapsedTime, start, stop); - printf("Elapsed time: %f ms\n", elapsedTime); -#endif -} diff --git a/cccl_upstream/cudax/examples/stf/jacobi_update_cond.cu b/cccl_upstream/cudax/examples/stf/jacobi_update_cond.cu deleted file mode 100644 index 5b6da267..00000000 --- a/cccl_upstream/cudax/examples/stf/jacobi_update_cond.cu +++ /dev/null @@ -1,101 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Jacobi method using the update_cond helper for clean condition management - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) -{ -#if _CCCL_CTK_BELOW(12, 4) - fprintf(stderr, "Waiving test: conditional nodes are only available since CUDA 12.4.\n"); - return 0; -#else - stackable_ctx ctx; - - size_t n = 4096; - size_t m = 4096; - double tol = 0.5; - int max_iter = 1000; - - if (argc > 2) - { - n = atol(argv[1]); - m = atol(argv[2]); - } - - if (argc > 3) - { - tol = atof(argv[3]); - } - - if (argc > 4) - { - max_iter = atoi(argv[4]); - } - - auto lA = ctx.logical_data(shape_of>(m, n)); - auto lAnew = ctx.logical_data(lA.shape()); - auto lresidual = ctx.logical_data(shape_of>()); - auto liter = ctx.logical_data(shape_of>()); - - ctx.parallel_for(lA.shape(), lA.write(), lAnew.write()).set_symbol("init")->* - [=] __device__(size_t i, size_t j, auto A, auto Anew) { - A(i, j) = (i == j) ? 10.0 : -1.0; - Anew(i, j) = A(i, j); - }; - - // Initialize iteration counter - ctx.parallel_for(box(1), liter.write())->*[] __device__(size_t, auto iter) { - *iter = 0; - }; - - { - auto while_guard = ctx.while_graph_scope(); - - ctx.parallel_for(inner<1>(lA.shape()), lA.read(), lAnew.rw(), lresidual.reduce(reducer::maxval())) - ->*[tol] __device__(size_t i, size_t j, auto A, auto Anew, auto& residual) { - Anew(i, j) = 0.25 * (A(i - 1, j) + A(i + 1, j) + A(i, j - 1) + A(i, j + 1)); - double error = fabs(A(i, j) - Anew(i, j)); - residual = ::std::max(error, residual); - }; - - ctx.parallel_for(inner<1>(lA.shape()), lA.rw(), lAnew.read())->*[] __device__(size_t i, size_t j, auto A, auto Anew) { - A(i, j) = Anew(i, j); - }; - - while_guard.update_cond(lresidual.read(), liter.rw())->*[tol, max_iter] __device__(auto residual, auto iter) { - bool converged = (*residual < tol); - bool max_reached = ((*iter)++ >= max_iter); // Maximum iteration limit - return !converged && !max_reached; // Continue if not converged and under limit - }; - } - - int final_iterations = ctx.wait(liter); - double final_residual = ctx.wait(lresidual); - - printf("Converged after %d iterations, residual = %lf\n", final_iterations, final_residual); - - ctx.finalize(); - - EXPECT(final_residual <= tol); - EXPECT(final_iterations < max_iter); - - return 0; -#endif -} diff --git a/cccl_upstream/cudax/examples/stf/launch_histogram.cu b/cccl_upstream/cudax/examples/stf/launch_histogram.cu deleted file mode 100644 index 8bc4c26b..00000000 --- a/cccl_upstream/cudax/examples/stf/launch_histogram.cu +++ /dev/null @@ -1,170 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief A naive parallel histogram algorithm written with launch - * - */ - -#include - -using namespace cuda::experimental::stf; - -__host__ __device__ double X0(int i) -{ - return sin((double) i); -} - -int main(int argc, char** argv) -{ - stream_ctx ctx; - - double lower_level = -1.0; - double upper_level = 1.0; - constexpr size_t num_levels = 21; - - size_t N = 128 * 1024UL; - if (argc > 1) - { - N = size_t(atoll(argv[1])); - } - - int check = 1; - if (argc > 2) - { - check = atoi(argv[2]); - } - - // fprintf(stderr, "SIZE %s\n", pretty_print_bytes(N * sizeof(double)).c_str()); - - std::vector X(N); - std::vector histo(num_levels - 1); - - for (size_t i = 0; i < N; i++) - { - X[i] = X0(i); - } - - // If we were to register each part one by one, there could be pages which - // cross multiple parts, and the pinning operation would fail. - cuda_safe_call(cudaHostRegister(&X[0], N * sizeof(double), cudaHostRegisterPortable)); - - auto lX = ctx.logical_data(&X[0], N); - lX.set_symbol("X"); - - auto lhisto = ctx.logical_data(&histo[0], num_levels - 1); - lhisto.set_symbol("histogram"); - - cuda_safe_call(cudaStreamSynchronize(ctx.fence())); - - cudaEvent_t start, stop; - cuda_safe_call(cudaEventCreate(&start)); - cuda_safe_call(cudaEventCreate(&stop)); - cuda_safe_call(cudaEventRecord(start, ctx.fence())); - - constexpr size_t BLOCK_THREADS = 128; - - // size_t NDEVS = 1; - // auto where = exec_place::repeat(exec_place::current_device(), NDEVS); - auto where = exec_place::current_device(); - - auto spec = con<8>(con(BLOCK_THREADS, mem((num_levels - 1) * sizeof(size_t)))); - - ctx.launch(spec, where, lX.read(), lhisto.write())->*[=] _CCCL_DEVICE(auto th, auto x, auto histo) { - size_t block_id = th.rank(0); - - slice smem_hist = th.template storage(1); - assert(smem_hist.size() == (num_levels - 1)); - - /* Thread local histogram */ - size_t local_hist[num_levels - 1]; - for (size_t k = 0; k < num_levels - 1; k++) - { - local_hist[k] = 0; - smem_hist[k] = 0; - } - - if (th.rank() == 0) - { - for (size_t k = 0; k < num_levels - 1; k++) - { - histo[k] = 0; - } - } - - for (size_t i = th.rank(); i < x.size(); i += th.size()) - { - double xi = x(i); - if (xi >= lower_level && xi < upper_level) - { - size_t bin = size_t(((num_levels - 1) * (xi - lower_level)) / (upper_level - lower_level)); - local_hist[bin]++; - } - } - - // smem was zero'ed - th.inner().sync(); - - /* Each thread contributes to an histogram in shared memory */ - for (size_t k = 0; k < num_levels - 1; k++) - { - atomicAdd((unsigned long long*) &smem_hist[k], local_hist[k]); - } - - // histo was zero'ed - th.sync(); - - if (th.inner().rank() == 0) - { - for (size_t k = 0; k < num_levels - 1; k++) - { - atomicAdd((unsigned long long*) &histo[k], smem_hist[k]); - } - } - }; - - cuda_safe_call(cudaEventRecord(stop, ctx.fence())); - - ctx.finalize(); - - float ms = 0; - cuda_safe_call(cudaEventElapsedTime(&ms, start, stop)); - - // fprintf(stdout, "%zu %f ms\n", N / 1024 / 1024, ms); - - if (check) - { - // fprintf(stderr, "Checking result...\n"); - size_t refhist[num_levels - 1]; - for (size_t i = 0; i < num_levels - 1; i++) - { - refhist[i] = 0; - } - - for (size_t i = 0; i < N; i++) - { - double xi = X[i]; - if (xi >= lower_level && xi < upper_level) - { - size_t bin = size_t(((num_levels - 1) * (xi - lower_level)) / (upper_level - lower_level)); - refhist[bin]++; - } - } - - // double dlevel = (upper_level - lower_level) / (num_levels - 1); - for (size_t i = 0; i < num_levels - 1; i++) - { - EXPECT(refhist[i] == histo[i]); - // fprintf(stderr, "[%lf:%lf[ %ld\n", lower_level + i * dlevel, lower_level + (i + 1) * dlevel, histo[i]); - } - } -} diff --git a/cccl_upstream/cudax/examples/stf/launch_scan.cu b/cccl_upstream/cudax/examples/stf/launch_scan.cu deleted file mode 100644 index d735e832..00000000 --- a/cccl_upstream/cudax/examples/stf/launch_scan.cu +++ /dev/null @@ -1,162 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief A parallel scan algorithm - * - */ - -#include // or equivalently - -#include - -using namespace cuda::experimental::stf; - -__host__ __device__ double X0(int) -{ - // return sin((double) i); - return 1.0; -} - -int main(int argc, char** argv) -{ - stream_ctx ctx; - // graph_ctx ctx; - - size_t N = 128 * 1024UL * 1024UL; - if (argc > 1) - { - N = size_t(atoll(argv[1])); - } - - int check = 0; - if (argc > 2) - { - check = atoi(argv[2]); - } - - std::vector X(N); - - for (size_t i = 0; i < N; i++) - { - X[i] = X0(i); - } - - auto lX = ctx.logical_data(&X[0], N); - - // No need to move this back to the host if we do not check the result - if (!check) - { - lX.set_write_back(false); - } - - cuda_safe_call(cudaStreamSynchronize(ctx.fence())); - - cudaEvent_t start, stop; - cuda_safe_call(cudaEventCreate(&start)); - cuda_safe_call(cudaEventCreate(&stop)); - cuda_safe_call(cudaEventRecord(start, ctx.fence())); - - constexpr size_t BLOCK_THREADS = 128; - constexpr size_t NBLOCKS = 8; - - auto spec = con(con(), mem(NBLOCKS * sizeof(double))); - - // auto where = exec_place::repeat(exec_place::current_device(), NDEVS); - auto where = exec_place::current_device(); - - ctx.launch(spec, where, lX.rw())->*[=] _CCCL_DEVICE(auto th, auto x) { - const size_t block_id = th.rank(0); - const size_t tid = th.inner().rank(); - // const size_t tid = th.rank(1, 0); - - // Block-wide partials using static allocation - __shared__ double block_partial_sum[th.static_width(1)]; - - // Device-wide partial sums - slice dev_partial_sum = th.template storage(0); - - /* Thread local prefix-sum */ - const box<1> b = th.apply_partition(shape(x), std::tuple()); - for (size_t i = b.get_begin(0) + 1; i < b.get_end(0); i++) - { - x(i) += x(i - 1); - } - block_partial_sum[tid] = x(b.get_end(0) - 1); - - th.inner().sync(); - - /* Block level : get partials sum across the different threads */ - if (tid == 0) - { // rank in scope block is 0 - // Prefix sum on partial sums - for (size_t i = 1; i < BLOCK_THREADS; i++) - { - block_partial_sum[i] += block_partial_sum[i - 1]; - } - dev_partial_sum[block_id] = block_partial_sum[BLOCK_THREADS - 1]; - } - - /* Reduce partial sums at device level : get sum across all blocks */ - th.sync(); - - if (block_id == 0 && tid == 0) - { // rank in scope 0 - for (size_t i = 1; i < NBLOCKS; i++) - { - dev_partial_sum[i] += dev_partial_sum[i - 1]; - // printf("SUMMED dev_partial_sum[%ld] = %f\n", i, dev_partial_sum[i]); - } - } - - th.sync(); - - for (size_t i = b.get_begin(0); i < b.get_end(0); i++) - { - if (tid > 0) - { - x(i) += block_partial_sum[tid - 1]; - } - - if (block_id > 0) - { - x(i) += dev_partial_sum[block_id - 1]; - } - } - }; - - cuda_safe_call(cudaEventRecord(stop, ctx.fence())); - - ctx.finalize(); - - float ms = 0; - cuda_safe_call(cudaEventElapsedTime(&ms, start, stop)); - - printf("%s in %f ms (%g GB/s)\n", - pretty_print_bytes(N * sizeof(double)).c_str(), - ms, - double(N * sizeof(double) / 1024 / 1024) / ms); - - if (check) - { - fprintf(stderr, "Checking result...\n"); - EXPECT(fabs(X[0] - X0(0)) < 0.00001); - for (size_t i = 0; i < N; i++) - { - if (fabs(X[i] - X[i - 1] - X0(i)) > 0.00001) - { - fprintf(stderr, "I %zu X[i] %f (X[i] - X[i-1]) %f expect %f\n", i, X[i], (X[i] - X[i - 1]), X0(i)); - } - EXPECT(fabs(X[i] - X[i - 1] - X0(i)) < 0.00001); - } - } -} diff --git a/cccl_upstream/cudax/examples/stf/launch_sum.cu b/cccl_upstream/cudax/examples/stf/launch_sum.cu deleted file mode 100644 index 1cdd2cf7..00000000 --- a/cccl_upstream/cudax/examples/stf/launch_sum.cu +++ /dev/null @@ -1,81 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief A reduction kernel written using launch - */ - -#include - -using namespace cuda::experimental::stf; - -double X0(int i) -{ - return sin((double) i); -} - -int main() -{ - context ctx; - - const size_t N = 128 * 1024 * 1024; - - std::vector X(N); - double sum = 0.0; - - double ref_sum = 0.0; - - for (size_t ind = 0; ind < N; ind++) - { - X[ind] = sin((double) ind); - ref_sum += X[ind]; - } - - auto lX = ctx.logical_data(&X[0], {N}); - auto lsum = ctx.logical_data(&sum, {1}); - - auto number_devices = 1; // - auto where = exec_place::repeat(exec_place::device(0), number_devices); - - auto spec = par<16>(con<32>()); - - ctx.launch(spec, where, lX.read(), lsum.rw())->*[] _CCCL_DEVICE(auto th, auto x, auto sum) { - // Each thread computes the sum of elements assigned to it - double local_sum = 0.0; - for (size_t i = th.rank(); i < x.size(); i += th.size()) - { - local_sum += x(i); - } - - auto ti = th.inner(); - - __shared__ double block_sum[th.static_width(1)]; - block_sum[ti.rank()] = local_sum; - - for (size_t s = ti.size() / 2; s > 0; s /= 2) - { - ti.sync(); - if (ti.rank() < s) - { - block_sum[ti.rank()] += block_sum[ti.rank() + s]; - } - } - - if (ti.rank() == 0) - { - atomicAdd(&sum(0), block_sum[0]); - } - }; - - ctx.finalize(); - - EXPECT(fabs(sum - ref_sum) < 0.0001); -} diff --git a/cccl_upstream/cudax/examples/stf/launch_sum_cub.cu b/cccl_upstream/cudax/examples/stf/launch_sum_cub.cu deleted file mode 100644 index a9e64dbb..00000000 --- a/cccl_upstream/cudax/examples/stf/launch_sum_cub.cu +++ /dev/null @@ -1,73 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief A reduction kernel written using launch and CUB - */ - -#include - -#include - -using namespace cuda::experimental::stf; - -double X0(int i) -{ - return sin((double) i); -} - -int main() -{ - context ctx; - - const size_t N = 128 * 1024 * 1024; - - std::vector X(N); - double sum = 0.0; - - double ref_sum = 0.0; - - for (size_t ind = 0; ind < N; ind++) - { - X[ind] = sin((double) ind); - ref_sum += X[ind]; - } - - auto lX = ctx.logical_data(&X[0], {N}); - auto lsum = ctx.logical_data(&sum, {1}); - - auto number_devices = 2; - auto where = exec_place::repeat(exec_place::device(0), number_devices); - - auto spec = par<32>(con<128>()); - - ctx.launch(spec, where, lX.read(), lsum.rw())->*[] _CCCL_DEVICE(auto th, auto x, auto sum) { - // Each thread computes the sum of elements assigned to it - double local_sum = 0.0; - for (auto ind : th.apply_partition(shape(x))) - { - local_sum += x(ind); - } - - using BlockReduce = cub::BlockReduce; - __shared__ typename BlockReduce::TempStorage temp_storage; - - double block_sum = BlockReduce(temp_storage).Sum(local_sum); - if (th.inner().rank() == 0) - { - atomicAdd(&sum(0), block_sum); - } - }; - - ctx.finalize(); - - EXPECT(fabs(sum - ref_sum) < 0.0001); -} diff --git a/cccl_upstream/cudax/examples/stf/linear_algebra/06-pdgemm-stackable.cu b/cccl_upstream/cudax/examples/stf/linear_algebra/06-pdgemm-stackable.cu deleted file mode 100644 index 649af6a8..00000000 --- a/cccl_upstream/cudax/examples/stf/linear_algebra/06-pdgemm-stackable.cu +++ /dev/null @@ -1,484 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief An example that implements a tiled matrix product over multiple devices using CUBLAS - */ - -#include - -#include "cuda/experimental/stf.cuh" -#include - -#define TILED - -using namespace cuda::experimental::stf; - -static std::unordered_map cublas_handles; - -/* Get a CUBLAS handle valid on the current device, or initialize it lazily */ -cublasHandle_t get_cublas_handle() -{ - int dev; - cuda_safe_call(cudaGetDevice(&dev)); - - auto& result = cublas_handles[dev]; - if (result == cublasHandle_t()) - { // not found, default value inserted - // Lazy initialization, and save the handle for future use - cuda_safe_call(cublasCreate(&result)); - } - return result; -} - -template -class matrix -{ -public: - matrix(stackable_ctx& ctx, - size_t NROWS, - size_t NCOLS, - size_t BLOCKSIZE_ROWS, - size_t BLOCKSIZE_COLS, - const char* _symbol = "matrix") - : h_array(nullptr) - , m(NROWS) - , n(NCOLS) - , mb(BLOCKSIZE_ROWS) - , nb(BLOCKSIZE_COLS) - , mt(0) - , nt(0) - , symbol(_symbol) - , ndevs(0) - , grid_p(0) - , grid_q(0) - { - assert(m % mb == 0); - assert(n % nb == 0); - - const size_t s = m * n * sizeof(T); - // cuda_safe_call(cudaMallocHost(&h_array, m*n*sizeof(T))); - // fprintf(stderr, "Allocating %ld x %ld x %ld = %ld bytes (%f GB) on host for %s\n", m, n, sizeof(T), s, - // s / (1024.0 * 1024.0 * 1024.0), _symbol); - h_array = static_cast(malloc(s)); - assert(h_array); - cuda_safe_call(cudaHostRegister(h_array, s, cudaHostRegisterPortable)); - - // Compute the number of blocks - mt = m / mb; - nt = n / nb; - - handles.resize(mt * nt); - - for (size_t colb = 0; colb < nt; colb++) - { - for (size_t rowb = 0; rowb < mt; rowb++) - { - T* addr_h = get_block_h(rowb, colb); - -#ifdef TILED - // tiles are stored contiguously - const size_t ld = mb; -#else - const size_t ld = m; -#endif - - std::ignore = ld; // avoid warning #177-D: variable "ld" was declared but never referenced - auto s = make_slice(addr_h, std::tuple{mb, nb}, ld); - auto tile = ctx.logical_data(s); - tile.set_write_back(false); - - tile.set_symbol(std::string(symbol) + "_" + std::to_string(rowb) + "_" + std::to_string(colb)); - - handles[rowb + colb * mt] = std::move(tile); - } - } - - cuda_safe_call(cudaGetDeviceCount(&ndevs)); - for (int a = 1; a * a <= ndevs; a++) - { - if (ndevs % a == 0) - { - grid_p = a; - grid_q = ndevs / a; - } - } - - assert(grid_p * grid_q == ndevs); - - // std::cout << "FOUND " << ndevs << " DEVICES " - // << "p=" << grid_p << " q=" << grid_q << '\n'; - } - - ~matrix() - { - if (h_array) - { - cuda_safe_call(cudaHostUnregister(h_array)); - free(h_array); - } - } - - // Disable copy and move operations - this is a resource-owning class used locally - matrix(const matrix&) = delete; - matrix& operator=(const matrix&) = delete; - matrix(matrix&&) = delete; - matrix& operator=(matrix&&) = delete; - - void push(access_mode mode) - { - for (auto& h : handles) - { - h.push(mode); - } - } - - int get_preferred_devid(int row, int col) const - { - return (row % grid_p) + (col % grid_q) * grid_p; - } - - auto& get_handle(int row, int col) - { - return handles[row + col * mt]; - } - - auto& get_handle(int row, int col) const - { - return handles[row + col * mt]; - } - - size_t get_index(size_t row, size_t col) const - { -#ifdef TILED - // Find which tile contains this element - const int tile_row = static_cast(row / mb); - const int tile_col = static_cast(col / nb); - - const size_t tile_size = mb * nb; - - // Look for the index of the beginning of the tile - const size_t tile_start = (tile_row + mt * tile_col) * tile_size; - - // Offset within the tile - const size_t offset = (row % mb) + (col % nb) * mb; - - return tile_start + offset; -#else - return row + col * m; -#endif - } - - T* get_block_h(int brow, int bcol) - { - const size_t index = get_index(brow * mb, bcol * nb); - return &h_array[index]; - } - - // Fill with func(Matrix*,row, col) - template - void fill(stackable_ctx& ctx, Fun&& fun) - { - nvtxRangePushA("FILL"); - // Fill blocks by blocks - for (size_t colb = 0; colb < nt; colb++) - { - for (size_t rowb = 0; rowb < mt; rowb++) - { - // Each task fills a block - auto& h = get_handle(rowb, colb); - int devid = get_preferred_devid(rowb, colb); - - ctx.parallel_for(exec_place::device(devid), h.shape(), h.write()).set_symbol("INIT")->* - [=] _CCCL_DEVICE(size_t lrow, size_t lcol, auto sA) { - const size_t row = lrow + rowb * sA.extent(0); - const size_t col = lcol + colb * sA.extent(1); - sA(lrow, lcol) = fun(row, col); - }; - } - } - nvtxRangePop(); - } - - T* h_array; - size_t m; // nrows - size_t n; // ncols - - size_t mb; // block size (rows) - size_t nb; // block size (cols) - - size_t mt; // numter of column blocks - size_t nt; // numter of row blocks - - // abstract data handles - std::vector>> handles; - - const char* symbol; - - // for the mapping - int ndevs; - int grid_p, grid_q; -}; - -void DGEMM( - stackable_ctx& ctx, - cublasOperation_t transa, - cublasOperation_t transb, - double alpha, - const matrix& A, - int A_row, - int A_col, - const matrix& B, - int B_row, - int B_col, - double beta, - matrix& C, - int C_row, - int C_col) -{ - const auto dev = exec_place::device(C.get_preferred_devid(C_row, C_col)); - - auto t = ctx.task( - dev, A.get_handle(A_row, A_col).read(), B.get_handle(B_row, B_col).read(), C.get_handle(C_row, C_col).rw()); - t.set_symbol("DGEMM"); - - t->*[&](cudaStream_t stream, auto tA, auto tB, auto tC) { - cuda_safe_call(cublasSetStream(get_cublas_handle(), stream)); - int k = tA.extent(transa == CUBLAS_OP_N ? 1 : 0); - cuda_safe_call(cublasDgemm( - get_cublas_handle(), - transa, - transb, - tC.extent(0), - tC.extent(1), - k, - &alpha, - tA.data_handle(), - tA.stride(1), - tB.data_handle(), - tB.stride(1), - &beta, - tC.data_handle(), - tC.stride(1))); - }; -} - -void PDGEMM(stackable_ctx& ctx, - cublasOperation_t transa, - cublasOperation_t transb, - double alpha, - const matrix& A, - const matrix& B, - double beta, - matrix& C) -{ - for (size_t m = 0; m < C.mt; m++) - { - for (size_t n = 0; n < C.nt; n++) - { - //========================================= - // alpha*A*B does not contribute; scale C - //========================================= - const size_t inner_k = transa == CUBLAS_OP_N ? A.n : A.m; - if (alpha == 0.0 || inner_k == 0) - { - DGEMM(ctx, transa, transb, alpha, A, 0, 0, B, 0, 0, beta, C, static_cast(m), static_cast(n)); - } - else if (transa == CUBLAS_OP_N) - { - //================================ - // CUBLAS_OP_N / CUBLAS_OP_N - //================================ - if (transb == CUBLAS_OP_N) - { - assert(A.nt == B.mt); - for (size_t k = 0; k < A.nt; k++) - { - const double zbeta = k == 0 ? beta : 1.0; - DGEMM(ctx, - transa, - transb, - alpha, - A, - static_cast(m), - static_cast(k), - B, - static_cast(k), - static_cast(n), - zbeta, - C, - static_cast(m), - static_cast(n)); - } - } - //===================================== - // CUBLAS_OP_N / CUBLAS_OP_T - //===================================== - else - { - for (size_t k = 0; k < A.nt; k++) - { - const double zbeta = k == 0 ? beta : 1.0; - DGEMM(ctx, - transa, - transb, - alpha, - A, - static_cast(m), - static_cast(k), - B, - static_cast(n), - static_cast(k), - zbeta, - C, - static_cast(m), - static_cast(n)); - } - } - } - else - { - //===================================== - // CUBLAS_OP_T / CUBLAS_OP_N - //===================================== - if (transb == CUBLAS_OP_N) - { - for (size_t k = 0; k < A.mt; k++) - { - const double zbeta = k == 0 ? beta : 1.0; - DGEMM(ctx, - transa, - transb, - alpha, - A, - static_cast(k), - static_cast(m), - B, - static_cast(k), - static_cast(n), - zbeta, - C, - static_cast(m), - static_cast(n)); - } - } - //========================================== - // CUBLAS_OP_T / CUBLAS_OP_T - //========================================== - else - { - for (size_t k = 0; k < A.mt; k++) - { - const double zbeta = k == 0 ? beta : 1.0; - DGEMM(ctx, - transa, - transb, - alpha, - A, - static_cast(k), - static_cast(m), - B, - static_cast(n), - static_cast(k), - zbeta, - C, - static_cast(m), - static_cast(n)); - } - } - } - } - } -} - -void run(stackable_ctx& ctx, size_t N, size_t NB) -{ - /// auto fixed_alloc = block_allocator(ctx, NB * NB * sizeof(double)); - // ctx.set_allocator(fixed_alloc); - - // Set up CUBLAS and CUSOLVER - int ndevs; - cuda_safe_call(cudaGetDeviceCount(&ndevs)); - - /* Warm up allocators */ - for (int d = 0; d < ndevs; d++) - { - auto lX = ctx.logical_data(shape_of>(1)); - ctx.parallel_for(exec_place::device(d), lX.shape(), lX.write())->*[] _CCCL_DEVICE(size_t, auto) {}; - } - - /* Initializes CUBLAS on all devices */ - for (int d = 0; d < ndevs; d++) - { - cuda_safe_call(cudaSetDevice(d)); - get_cublas_handle(); - } - - matrix A(ctx, N, N, NB, NB, "A"); - matrix B(ctx, N, N, NB, NB, "B"); - matrix C(ctx, N, N, NB, NB, "C"); - - // (Hilbert matrix + 2*N*Id) - auto hilbert = [=] _CCCL_HOST_DEVICE(size_t row, size_t col) { - return 1.0 / (col + row + 1.0) + 2.0 * N * (col == row); - }; - - A.fill(ctx, hilbert); - B.fill(ctx, hilbert); - C.fill(ctx, hilbert); - - cudaEvent_t startEvent, stopEvent; - - cuda_safe_call(cudaEventCreate(&startEvent)); - cuda_safe_call(cudaEventCreate(&stopEvent)); - - cuda_safe_call(cudaEventRecord(startEvent, ctx.fence())); - - ctx.push(); - A.push(access_mode::read); - B.push(access_mode::read); - C.push(access_mode::rw); - PDGEMM(ctx, CUBLAS_OP_N, CUBLAS_OP_N, 1.0, A, B, -2.0, C); - ctx.pop(); - - cuda_safe_call(cudaEventRecord(stopEvent, ctx.fence())); - - ctx.finalize(); - - float milliseconds; - cuda_safe_call(cudaEventElapsedTime(&milliseconds, startEvent, stopEvent)); - - const double gflops_pdgemm = - 2.0 * (static_cast(N) * static_cast(N) * static_cast(N)) / 1000000000.0; - ::std::cout - << "[PDDGEMM] ELAPSED: " << milliseconds << " ms, GFLOPS: " << gflops_pdgemm / (milliseconds / 1000.0) << '\n'; -} - -int main(int argc, char** argv) -{ - size_t N = 4096; - size_t NB = 512; - - if (argc > 1) - { - N = static_cast(::std::atoi(argv[1])); - } - - if (argc > 2) - { - NB = static_cast(::std::atoi(argv[2])); - } - - assert(N % NB == 0); - - stackable_ctx ctx; - run(ctx, N, NB); -} diff --git a/cccl_upstream/cudax/examples/stf/linear_algebra/06-pdgemm.cu b/cccl_upstream/cudax/examples/stf/linear_algebra/06-pdgemm.cu deleted file mode 100644 index c6aa5450..00000000 --- a/cccl_upstream/cudax/examples/stf/linear_algebra/06-pdgemm.cu +++ /dev/null @@ -1,399 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief An example that implements a tiled matrix product over multiple devices using CUBLAS - * - * This also illustrates how the same code base can be used both with a - * stream_ctx and a graph_ctx backend. - */ - -#include -#include - -#define TILED - -using namespace cuda::experimental::stf; - -static std::unordered_map> cublas_handles; - -/* Get a CUBLAS handle valid on the current execution place, or initialize it lazily */ -cublasHandle_t get_cublas_handle(const exec_place& ep = exec_place::current_device()) -{ - auto& result = cublas_handles[ep]; - if (result == cublasHandle_t()) - { // not found, default value inserted - // Lazy initialization, and save the handle for future use - cuda_safe_call(cublasCreate(&result)); - } - return result; -} - -template -class matrix -{ -public: - matrix(stream_ctx& ctx, - size_t NROWS, - size_t NCOLS, - size_t BLOCKSIZE_ROWS, - size_t BLOCKSIZE_COLS, - const char* _symbol = "matrix") - { - symbol = _symbol; - - m = NROWS; - mb = BLOCKSIZE_ROWS; - - n = NCOLS; - nb = BLOCKSIZE_COLS; - - assert(m % mb == 0); - assert(n % nb == 0); - - size_t s = ((size_t) m) * ((size_t) n) * sizeof(T); - // cuda_safe_call(cudaMallocHost(&h_array, m*n*sizeof(T))); - // fprintf(stderr, "Allocating %ld x %ld x %ld = %ld bytes (%f GB) on host for %s\n", m, n, sizeof(T), s, - // s / (1024.0 * 1024.0 * 1024.0), _symbol); - h_array = (T*) malloc(s); - assert(h_array); - cuda_safe_call(cudaHostRegister(h_array, s, cudaHostRegisterPortable)); - - // Compute the number of blocks - mt = m / mb; - nt = n / nb; - - handles.resize(mt * nt); - - for (size_t colb = 0; colb < nt; colb++) - { - for (size_t rowb = 0; rowb < mt; rowb++) - { - T* addr_h = get_block_h(rowb, colb); - -#ifdef TILED - // tiles are stored contiguously - const size_t ld = mb; -#else - const size_t ld = m; -#endif - - std::ignore = ld; // avoid warning #177-D: variable "ld" was declared but never referenced - auto s = make_slice(addr_h, std::tuple{mb, nb}, ld); - auto tile = ctx.logical_data(s); - tile.set_write_back(false); - - tile.set_symbol(std::string(symbol) + "_" + std::to_string(rowb) + "_" + std::to_string(colb)); - - handles[rowb + colb * mt] = std::move(tile); - } - } - - cuda_safe_call(cudaGetDeviceCount(&ndevs)); - for (int a = 1; a * a <= ndevs; a++) - { - if (ndevs % a == 0) - { - grid_p = a; - grid_q = ndevs / a; - } - } - - assert(grid_p * grid_q == ndevs); - - // std::cout << "FOUND " << ndevs << " DEVICES " - // << "p=" << grid_p << " q=" << grid_q << '\n'; - } - - int get_preferred_devid(int row, int col) - { - return (row % grid_p) + (col % grid_q) * grid_p; - } - - logical_data>& get_handle(int row, int col) - { - return handles[row + col * mt]; - } - - size_t get_index(size_t row, size_t col) - { -#ifdef TILED - // Find which tile contains this element - int tile_row = row / mb; - int tile_col = col / nb; - - size_t tile_size = mb * nb; - - // Look for the index of the beginning of the tile - size_t tile_start = (tile_row + mt * tile_col) * tile_size; - - // Offset within the tile - size_t offset = (row % mb) + (col % nb) * mb; - - return tile_start + offset; -#else - return row + col * m; -#endif - } - - T* get_block_h(int brow, int bcol) - { - size_t index = get_index(brow * mb, bcol * nb); - return &h_array[index]; - } - - // Fill with func(Matrix*,row, col) - template - void fill(stream_ctx& ctx, Fun&& fun) - { - nvtx_range r("fill"); - - // Fill blocks by blocks - for (size_t colb = 0; colb < nt; colb++) - { - for (size_t rowb = 0; rowb < mt; rowb++) - { - // Each task fills a block - auto& h = get_handle(rowb, colb); - int devid = get_preferred_devid(rowb, colb); - - ctx.parallel_for(exec_place::device(devid), h.shape(), h.write()).set_symbol("INIT")->* - [=] _CCCL_DEVICE(size_t lrow, size_t lcol, auto sA) { - size_t row = lrow + rowb * sA.extent(0); - size_t col = lcol + colb * sA.extent(1); - sA(lrow, lcol) = fun(row, col); - }; - } - } - } - - T* h_array; - size_t m; // nrows - size_t n; // ncols - - size_t mb; // block size (rows) - size_t nb; // block size (cols) - - size_t mt; // numter of column blocks - size_t nt; // numter of row blocks - - // abstract data handles - std::vector>> handles; - - const char* symbol; - - // for the mapping - int ndevs; - int grid_p, grid_q; -}; - -void DGEMM( - stream_ctx& ctx, - cublasOperation_t transa, - cublasOperation_t transb, - double alpha, - matrix& A, - int A_row, - int A_col, - matrix& B, - int B_row, - int B_col, - double beta, - matrix& C, - int C_row, - int C_col) -{ - auto dev = exec_place::device(C.get_preferred_devid(C_row, C_col)); - - auto t = ctx.task( - dev, A.get_handle(A_row, A_col).read(), B.get_handle(B_row, B_col).read(), C.get_handle(C_row, C_col).rw()); - t.set_symbol("DGEMM"); - - t->*[&](cudaStream_t stream, auto tA, auto tB, auto tC) { - cuda_safe_call(cublasSetStream(get_cublas_handle(), stream)); - int k = tA.extent(transa == CUBLAS_OP_N ? 1 : 0); - cuda_safe_call(cublasDgemm( - get_cublas_handle(), - transa, - transb, - tC.extent(0), - tC.extent(1), - k, - &alpha, - tA.data_handle(), - tA.stride(1), - tB.data_handle(), - tB.stride(1), - &beta, - tC.data_handle(), - tC.stride(1))); - }; -} - -void PDGEMM(stream_ctx& ctx, - cublasOperation_t transa, - cublasOperation_t transb, - double alpha, - matrix& A, - matrix& B, - double beta, - matrix& C) -{ - nvtx_range r("PDGEMM"); - - for (size_t m = 0; m < C.mt; m++) - { - for (size_t n = 0; n < C.nt; n++) - { - //========================================= - // alpha*A*B does not contribute; scale C - //========================================= - int inner_k = transa == CUBLAS_OP_N ? A.n : A.m; - if (alpha == 0.0 || inner_k == 0) - { - DGEMM(ctx, transa, transb, alpha, A, 0, 0, B, 0, 0, beta, C, m, n); - } - else if (transa == CUBLAS_OP_N) - { - //================================ - // CUBLAS_OP_N / CUBLAS_OP_N - //================================ - if (transb == CUBLAS_OP_N) - { - assert(A.nt == B.mt); - for (size_t k = 0; k < A.nt; k++) - { - double zbeta = k == 0 ? beta : 1.0; - DGEMM(ctx, transa, transb, alpha, A, m, k, B, k, n, zbeta, C, m, n); - } - } - //===================================== - // CUBLAS_OP_N / CUBLAS_OP_T - //===================================== - else - { - for (size_t k = 0; k < A.nt; k++) - { - double zbeta = k == 0 ? beta : 1.0; - DGEMM(ctx, transa, transb, alpha, A, m, k, B, n, k, zbeta, C, m, n); - } - } - } - else - { - //===================================== - // CUBLAS_OP_T / CUBLAS_OP_N - //===================================== - if (transb == CUBLAS_OP_N) - { - for (size_t k = 0; k < A.mt; k++) - { - double zbeta = k == 0 ? beta : 1.0; - DGEMM(ctx, transa, transb, alpha, A, k, m, B, k, n, zbeta, C, m, n); - } - } - //========================================== - // CUBLAS_OP_T / CUBLAS_OP_T - //========================================== - else - { - for (size_t k = 0; k < A.mt; k++) - { - double zbeta = k == 0 ? beta : 1.0; - DGEMM(ctx, transa, transb, alpha, A, k, m, B, n, k, zbeta, C, m, n); - } - } - } - } - } -} - -void run(stream_ctx& ctx, size_t N, size_t NB) -{ - auto fixed_alloc = block_allocator(ctx, NB * NB * sizeof(double)); - ctx.set_allocator(fixed_alloc); - - // Set up CUBLAS and CUSOLVER - int ndevs; - cuda_safe_call(cudaGetDeviceCount(&ndevs)); - - /* Warm up allocators */ - for (int d = 0; d < ndevs; d++) - { - auto lX = ctx.logical_data(shape_of>(1)); - ctx.parallel_for(exec_place::device(d), lX.shape(), lX.write())->*[] _CCCL_DEVICE(size_t, auto) {}; - } - - /* Initializes CUBLAS on all devices */ - for (int d = 0; d < ndevs; d++) - { - cuda_safe_call(cudaSetDevice(d)); - get_cublas_handle(); - } - - matrix A(ctx, N, N, NB, NB, "A"); - matrix B(ctx, N, N, NB, NB, "B"); - matrix C(ctx, N, N, NB, NB, "C"); - - // (Hilbert matrix + 2*N*Id) - auto hilbert = [=] _CCCL_HOST_DEVICE(size_t row, size_t col) { - return 1.0 / (col + row + 1.0) + 2.0 * N * (col == row); - }; - - A.fill(ctx, hilbert); - B.fill(ctx, hilbert); - C.fill(ctx, hilbert); - - cudaEvent_t startEvent, stopEvent; - - cuda_safe_call(cudaEventCreate(&startEvent)); - cuda_safe_call(cudaEventCreate(&stopEvent)); - - cuda_safe_call(cudaEventRecord(startEvent, ctx.fence())); - - PDGEMM(ctx, CUBLAS_OP_N, CUBLAS_OP_N, 1.0, A, B, -2.0, C); - - cuda_safe_call(cudaEventRecord(stopEvent, ctx.fence())); - - ctx.finalize(); - - float milliseconds; - cuda_safe_call(cudaEventElapsedTime(&milliseconds, startEvent, stopEvent)); - - double gflops_pdgemm = 2.0 * ((double) N * (double) N * (double) N) / (1000000000.0); - std::cout - << "[PDDGEMM] ELAPSED: " << milliseconds << " ms, GFLOPS: " << gflops_pdgemm / (milliseconds / 1000.0) << '\n'; -} - -int main(int argc, char** argv) -{ - size_t N = 4096; - size_t NB = 512; - - if (argc > 1) - { - N = atoi(argv[1]); - } - - if (argc > 2) - { - NB = atoi(argv[2]); - } - - assert(N % NB == 0); - - stream_ctx ctx; - run(ctx, N, NB); - - // // Also run using a graph context. - // ctx = graph_ctx(); - // run(ctx, N, NB); -} diff --git a/cccl_upstream/cudax/examples/stf/linear_algebra/07-cholesky.cu b/cccl_upstream/cudax/examples/stf/linear_algebra/07-cholesky.cu deleted file mode 100644 index c9d4d30e..00000000 --- a/cccl_upstream/cudax/examples/stf/linear_algebra/07-cholesky.cu +++ /dev/null @@ -1,748 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief This example implements a Cholesky decomposition over multiple devices using CUBLAS and CUSOLVER - * - * It also illustrates how we can use CUDASTF to allocate temporary data for CUSOLVER in CUDASTF tasks - */ - -#include -#include - -#include - -#include - -#define TILED - -using namespace cuda::experimental::stf; - -// Global for the sake of simplicity ! -stream_ctx ctx; - -/* Get a CUBLAS handle valid on the current execution place, or initialize it lazily */ -cublasHandle_t& get_cublas_handle(const exec_place& ep = exec_place::current_device()) -{ - static std::unordered_map> cublas_handles; - auto& result = cublas_handles[ep]; - if (result == cublasHandle_t()) - { // not found, default value inserted - // Lazy initialization, and save the handle for future use - cuda_safe_call(cublasCreate(&result)); - } - return result; -} - -/* Get a CUSOLVER handle valid on the current execution place, or initialize it lazily */ -cusolverDnHandle_t& get_cusolver_handle(const exec_place& ep = exec_place::current_device()) -{ - static std::unordered_map> cusolver_handles; - auto& result = cusolver_handles[ep]; - if (result == cusolverDnHandle_t()) - { // not found, default value inserted - // Lazy initialization, and save the handle for future use - cuda_safe_call(cusolverDnCreate(&result)); - } - return result; -} - -template -class matrix -{ -public: - matrix(int NROWS, int NCOLS, int BLOCKSIZE_ROWS, int BLOCKSIZE_COLS, bool is_sym, const char* _symbol = "matrix") - { - symbol = _symbol; - - sym_matrix = is_sym; - - m = NROWS; - mb = BLOCKSIZE_ROWS; - - n = NCOLS; - nb = BLOCKSIZE_COLS; - - assert(m % mb == 0); - assert(n % nb == 0); - - // cuda_safe_call(cudaMallocHost(&h_array, m*n*sizeof(T))); - // fprintf(stderr, "Allocating %ld x %ld x %ld = %ld bytes (%f GB) on host for %s\n", m, n, sizeof(T), s, - // s / (1024.0 * 1024.0 * 1024.0), _symbol); - h_array.resize(m * n); - cuda_safe_call(cudaHostRegister(&h_array[0], h_array.size() * sizeof(T), cudaHostRegisterPortable)); - - // Compute the number of blocks - mt = m / mb; - nt = n / nb; - - handles.resize(mt * nt); - - for (size_t colb = 0; colb < nt; colb++) - { - int low_rowb = sym_matrix ? colb : 0; - for (size_t rowb = low_rowb; rowb < mt; rowb++) - { - T* addr_h = get_block_h(rowb, colb); - auto& h = handle(rowb, colb); - -#ifdef TILED - // tiles are stored contiguously - size_t ld = mb; -#else - size_t ld = m; -#endif - std::ignore = ld; // work around bug in compiler - h = ctx.logical_data(make_slice(addr_h, std::tuple{mb, nb}, ld)); - h.set_symbol(std::string(symbol) + "_" + std::to_string(rowb) + "_" + std::to_string(colb)); - h.set_write_back(false); - } - } - - cuda_safe_call(cudaGetDeviceCount(&ndevs)); - for (int a = 1; a * a <= ndevs; a++) - { - if (ndevs % a == 0) - { - grid_p = a; - grid_q = ndevs / a; - } - } - - assert(grid_p * grid_q == ndevs); - - // std::cout << "FOUND " << ndevs << " DEVICES " - // << "p=" << grid_p << " q=" << grid_q << '\n'; - } - - int get_preferred_devid(int row, int col) - { - return (row % grid_p) + (col % grid_q) * grid_p; - } - - auto& handle(int row, int col) - { - return handles[row + col * mt]; - } - - size_t get_index(size_t row, size_t col) - { -#ifdef TILED - // Find which tile contains this element - int tile_row = row / mb; - int tile_col = col / nb; - - size_t tile_size = mb * nb; - - // Look for the index of the beginning of the tile - size_t tile_start = (tile_row + mt * tile_col) * tile_size; - - // Offset within the tile - size_t offset = (row % mb) + (col % nb) * mb; - - return tile_start + offset; -#else - return row + col * m; -#endif - } - - T* get_block_h(int brow, int bcol) - { - size_t index = get_index(brow * mb, bcol * nb); - return &h_array[index]; - } - - // Fill with func(Matrix*,row, col) - template - void fill(Fun&& fun) - { - nvtx_range r("fill"); - - // Fill blocks by blocks - for (size_t colb = 0; colb < nt; colb++) - { - size_t low_rowb = sym_matrix ? colb : 0; - for (size_t rowb = low_rowb; rowb < mt; rowb++) - { - // Each task fills a block - auto& h = handle(rowb, colb); - int devid = get_preferred_devid(rowb, colb); - - ctx.parallel_for(exec_place::device(devid), h.shape(), h.write()).set_symbol("INIT")->* - [=] _CCCL_DEVICE(size_t lrow, size_t lcol, auto sA) { - size_t row = lrow + rowb * sA.extent(0); - size_t col = lcol + colb * sA.extent(1); - sA(lrow, lcol) = fun(row, col); - }; - } - } - } - - std::vector h_array; - size_t m; // nrows - size_t n; // ncols - - // Is this a sym matrix ? (lower assumed) - bool sym_matrix; - - size_t mb; // block size (rows) - size_t nb; // block size (cols) - - size_t mt; // number of column blocks - size_t nt; // number of row blocks - - // abstract data handles - std::vector>> handles; - - const char* symbol; - - // for the mapping - int ndevs; - int grid_p, grid_q; -}; - -void DPOTRF(cublasFillMode_t uplo, class matrix& A, int A_row, int A_col) -{ - auto& Akk = A.handle(A_row, A_col); - size_t m_akk = Akk.shape().extent(0); - // Note that the handle may be different from the actual handle... - int Lwork_expected; - cuda_safe_call(cusolverDnDpotrf_bufferSize(get_cusolver_handle(), uplo, m_akk, nullptr, 0, &Lwork_expected)); - - auto potrf_buffer = ctx.logical_data(size_t(Lwork_expected)); - potrf_buffer.set_allocator(ctx.get_default_allocator()); - - auto devInfo = ctx.logical_data(shape_of>(1)); - - auto t = - ctx.task(exec_place::device(A.get_preferred_devid(A_row, A_col)), Akk.rw(), potrf_buffer.write(), devInfo.write()); - t.set_symbol("DPOTRF"); - t->*[uplo](cudaStream_t s, auto sAkk, auto buffer, auto info) { - auto& h = get_cusolver_handle(); - cuda_safe_call(cusolverDnSetStream(h, s)); - - cuda_safe_call(cusolverDnDpotrf( - h, - uplo, - sAkk.extent(0), - sAkk.data_handle(), - sAkk.stride(1), - buffer.data_handle(), - buffer.extent(0), - info.data_handle())); - }; -} - -void DGEMM( - cublasOperation_t transa, - cublasOperation_t transb, - double alpha, - class matrix& A, - int A_row, - int A_col, - class matrix& B, - int B_row, - int B_col, - double beta, - class matrix& C, - int C_row, - int C_col) -{ - auto t = ctx.task(exec_place::device(A.get_preferred_devid(C_row, C_col)), - A.handle(A_row, A_col).read(), - B.handle(B_row, B_col).read(), - C.handle(C_row, C_col).rw()); - t.set_symbol("DGEMM"); - t->*[transa, transb, alpha, beta](cudaStream_t s, auto sA, auto sB, auto sC) { - auto& h = get_cublas_handle(); - cuda_safe_call(cublasSetStream(h, s)); - - auto k = (transa == CUBLAS_OP_N) ? sA.extent(1) : sA.extent(0); - cuda_safe_call(cublasDgemm( - h, - transa, - transb, - sC.extent(0), - sC.extent(1), - k, - &alpha, - sA.data_handle(), - sA.stride(1), - sB.data_handle(), - sB.stride(1), - &beta, - sC.data_handle(), - sC.stride(1))); - }; -} - -void DSYRK( - cublasFillMode_t uplo, - cublasOperation_t trans, - double alpha, - class matrix& A, - int A_row, - int A_col, - double beta, - class matrix& C, - int C_row, - int C_col) -{ - auto t = ctx.task(exec_place::device(A.get_preferred_devid(C_row, C_col)), - A.handle(A_row, A_col).read(), - C.handle(C_row, C_col).rw()); - t.set_symbol("DSYRK"); - t->*[uplo, trans, alpha, beta](cudaStream_t s, auto sA, auto sC) { - auto& h = get_cublas_handle(); - cuda_safe_call(cublasSetStream(h, s)); - - // number of rows of matrix op(A) and C - auto n = sC.extent(0); - - // number of columns of matrix op(A) - auto k = (trans == CUBLAS_OP_N) ? sA.extent(1) : sA.extent(0); - - cuda_safe_call( - cublasDsyrk(h, uplo, trans, n, k, &alpha, sA.data_handle(), sA.stride(1), &beta, sC.data_handle(), sC.stride(1))); - }; -} - -void DTRSM( - cublasSideMode_t side, - cublasFillMode_t uplo, - cublasOperation_t transa, - cublasDiagType_t diag, - double alpha, - class matrix& A, - int A_row, - int A_col, - class matrix& B, - int B_row, - int B_col) -{ - auto t = ctx.task(exec_place::device(A.get_preferred_devid(B_row, B_col)), - A.handle(A_row, A_col).read(), - B.handle(B_row, B_col).rw()); - t.set_symbol("DTRSM"); - t->*[side, uplo, transa, diag, alpha](cudaStream_t s, auto sA, auto sB) { - auto& h = get_cublas_handle(); - cuda_safe_call(cublasSetStream(h, s)); - - cuda_safe_call(cublasDtrsm( - h, - side, - uplo, - transa, - diag, - sB.extent(0), - sB.extent(1), - &alpha, - sA.data_handle(), - sA.stride(1), - sB.data_handle(), - sB.stride(1))); - }; -} - -void PDNRM2_HOST(matrix* A, double* result) -{ -#ifdef HAVE_DOT - reserved::dot::set_current_color("red"); -#endif - - for (size_t rowb = 0; rowb < A->mt; rowb++) - { - for (size_t colb = 0; colb < A->nt; colb++) - { - ctx.host_launch(A->handle(rowb, colb).read())->*[=](auto sA) { - double res2 = 0.0; - for (size_t col = 0; col < sA.extent(1); col++) - { - for (size_t row = 0; row < sA.extent(0); row++) - { - double v = sA(row, col); - res2 += v * v; - } - } - *result += res2; - }; - } - } -} - -void PDPOTRF(matrix& A) -{ - auto guard = ctx.dot_section("PDPOTRF"); - -#ifdef HAVE_DOT - reserved::dot::set_current_color("yellow"); -#endif - - assert(A.m == A.n); - assert(A.mt == A.nt); - - int NBLOCKS = A.mt; - assert(A.mb == A.nb); - - cuda_safe_call(cudaSetDevice(0)); - - for (int K = 0; K < NBLOCKS; K++) - { - int dev_akk = A.get_preferred_devid(K, K); - cuda_safe_call(cudaSetDevice(A.get_preferred_devid(K, K))); - DPOTRF(CUBLAS_FILL_MODE_LOWER, A, K, K); - - for (int row = K + 1; row < NBLOCKS; row++) - { - cuda_safe_call(cudaSetDevice(A.get_preferred_devid(row, K))); - DTRSM(CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_LOWER, CUBLAS_OP_T, CUBLAS_DIAG_NON_UNIT, 1.0, A, K, K, A, row, K); - - for (int col = K + 1; col < row; col++) - { - cuda_safe_call(cudaSetDevice(A.get_preferred_devid(row, col))); - DGEMM(CUBLAS_OP_N, CUBLAS_OP_T, -1.0, A, row, K, A, col, K, 1.0, A, row, col); - } - - cuda_safe_call(cudaSetDevice(A.get_preferred_devid(row, row))); - DSYRK(CUBLAS_FILL_MODE_LOWER, CUBLAS_OP_N, -1.0, A, row, K, 1.0, A, row, row); - } - } - cuda_safe_call(cudaSetDevice(0)); -} - -// Algorithm from PLASMA -void PDTRSM(cublasSideMode_t side, - cublasFillMode_t uplo, - cublasOperation_t trans, - cublasDiagType_t diag, - double alpha, - class matrix& A, - class matrix& B) -{ - auto guard = ctx.dot_section("PDTRSM"); - - // std::cout << "[PDTRSM] START B MT " << B.mt << " NT " << B.nt << '\n'; - - if (side == CUBLAS_SIDE_LEFT) - { - if (uplo == CUBLAS_FILL_MODE_UPPER) - { - // TODO - assert(0); - abort(); - } - else - { - //=========================================== - // CUBLAS_SIDE_LEFT / CUBLAS_FILL_MODE_LOWER / CUBLAS_OP_N - //=========================================== - if (trans == CUBLAS_OP_N) - { - for (size_t k = 0; k < B.mt; k++) - { - double lalpha = k == 0 ? alpha : 1.0; - for (size_t n = 0; n < B.nt; n++) - { - cuda_safe_call(cudaSetDevice(A.get_preferred_devid(k, k))); - DTRSM(side, uplo, trans, diag, lalpha, A, k, k, B, k, n); - } - for (size_t m = k + 1; m < B.mt; m++) - { - for (size_t n = 0; n < B.nt; n++) - { - cuda_safe_call(cudaSetDevice(A.get_preferred_devid(m, k))); - DGEMM(CUBLAS_OP_N, CUBLAS_OP_N, -1.0, A, m, k, B, k, n, lalpha, B, m, n); - } - } - } - } - //================================================ - // CUBLAS_SIDE_LEFT / CUBLAS_FILL_MODE_LOWER / CUBLAS_OP_[C|T] - //================================================ - else - { - for (size_t k = 0; k < B.mt; k++) - { - double lalpha = k == 0 ? alpha : 1.0; - for (size_t n = 0; n < B.nt; n++) - { - cuda_safe_call(cudaSetDevice(A.get_preferred_devid(B.mt - k - 1, B.mt - k - 1))); - DTRSM(side, uplo, trans, diag, lalpha, A, B.mt - k - 1, B.mt - k - 1, B, B.mt - k - 1, n); - } - for (size_t m = k + 1; m < B.mt; m++) - { - for (size_t n = 0; n < B.nt; n++) - { - cuda_safe_call(cudaSetDevice(A.get_preferred_devid(B.mt - k - 1, B.mt - 1 - m))); - DGEMM( - trans, CUBLAS_OP_N, -1.0, A, B.mt - k - 1, B.mt - 1 - m, B, B.mt - k - 1, n, lalpha, B, B.mt - 1 - m, n); - } - } - } - } - } - } - else - { - // TODO - abort(); - } - cuda_safe_call(cudaSetDevice(0)); - // std::cout << "[PDTRSM] END" << '\n'; -} - -void PDPOTRS(matrix& A, class matrix& B, cublasFillMode_t uplo) -{ - auto guard = ctx.dot_section("PDPOTRS"); -#ifdef HAVE_DOT - reserved::dot::set_current_color("green"); -#endif - - // std::cout << "[PDPOTRS] START" << '\n'; - // Call the parallel functions. - PDTRSM( - CUBLAS_SIDE_LEFT, uplo, uplo == CUBLAS_FILL_MODE_UPPER ? CUBLAS_OP_T : CUBLAS_OP_N, CUBLAS_DIAG_NON_UNIT, 1.0, A, B); - -#ifdef HAVE_DOT - reserved::dot::set_current_color("darkgreen"); -#endif - - PDTRSM( - CUBLAS_SIDE_LEFT, uplo, uplo == CUBLAS_FILL_MODE_UPPER ? CUBLAS_OP_N : CUBLAS_OP_T, CUBLAS_DIAG_NON_UNIT, 1.0, A, B); - // std::cout << "[PDPOTRS] END" << '\n'; -} - -/***************************************************************************** - * Parallel tile matrix-matrix - *multiplication. - * @see plasma_omp_dgemm - ******************************************************************************/ -void PDGEMM(cublasOperation_t transa, - cublasOperation_t transb, - double alpha, - class matrix& A, - class matrix& B, - double beta, - class matrix& C) -{ - auto guard = ctx.dot_section("PDGEMM"); - -#ifdef HAVE_DOT - reserved::dot::set_current_color("blue"); -#endif - - for (size_t m = 0; m < C.mt; m++) - { - for (size_t n = 0; n < C.nt; n++) - { - //========================================= - // alpha*A*B does not contribute; scale C - //========================================= - int inner_k = transa == CUBLAS_OP_N ? A.n : A.m; - if (alpha == 0.0 || inner_k == 0) - { - DGEMM(transa, transb, alpha, A, 0, 0, B, 0, 0, beta, C, m, n); - } - else if (transa == CUBLAS_OP_N) - { - //================================ - // CUBLAS_OP_N / CUBLAS_OP_N - //================================ - if (transb == CUBLAS_OP_N) - { - for (size_t k = 0; k < A.nt; k++) - { - double zbeta = k == 0 ? beta : 1.0; - DGEMM(transa, transb, alpha, A, m, k, B, k, n, zbeta, C, m, n); - } - } - //===================================== - // CUBLAS_OP_N / CUBLAS_OP_T - //===================================== - else - { - for (size_t k = 0; k < A.nt; k++) - { - double zbeta = k == 0 ? beta : 1.0; - DGEMM(transa, transb, alpha, A, m, k, B, n, k, zbeta, C, m, n); - } - } - } - else - { - //===================================== - // CUBLAS_OP_T / CUBLAS_OP_N - //===================================== - if (transb == CUBLAS_OP_N) - { - for (size_t k = 0; k < A.mt; k++) - { - double zbeta = k == 0 ? beta : 1.0; - DGEMM(transa, transb, alpha, A, k, m, B, k, n, zbeta, C, m, n); - } - } - //========================================== - // CUBLAS_OP_T / CUBLAS_OP_T - //========================================== - else - { - for (size_t k = 0; k < A.mt; k++) - { - double zbeta = k == 0 ? beta : 1.0; - DGEMM(transa, transb, alpha, A, k, m, B, n, k, zbeta, C, m, n); - } - } - } - } - } -} - -int main(int argc, char** argv) -{ - int N = 1024; - int NB = 128; - - if (argc > 1) - { - N = atoi(argv[1]); - } - - if (argc > 2) - { - NB = atoi(argv[2]); - } - - int check_result = 1; - if (getenv("CHECK_RESULT")) - { - check_result = atoi(getenv("CHECK_RESULT")); - } - - assert(N % NB == 0); - - // Use pools of preallocated blocks - auto fixed_alloc = block_allocator(ctx, NB * NB * sizeof(double)); - ctx.set_allocator(fixed_alloc); - - // Set up CUBLAS and CUSOLVER - int ndevs; - cuda_safe_call(cudaGetDeviceCount(&ndevs)); - - for (int d = 0; d < ndevs; d++) - { - auto lX = ctx.logical_data(shape_of>(1)); - ctx.parallel_for(exec_place::device(d), lX.shape(), lX.write())->*[] _CCCL_DEVICE(size_t, auto) {}; - cuda_safe_call(cudaSetDevice(d)); - get_cublas_handle(); - get_cusolver_handle(); - } - - cuda_safe_call(cudaSetDevice(0)); - - matrix A(N, N, NB, NB, true, "A"); - matrix Aref(N, N, NB, NB, false, "Aref"); - - // (Hilbert matrix + 2*N*Id) to have a diagonal dominant matrix - auto hilbert = [=] _CCCL_HOST_DEVICE(size_t row, size_t col) { - return 1.0 / (col + row + 1.0) + 2.0 * N * (col == row); - }; - - auto s = ctx.dot_section("fillA"); - if (check_result) - { - Aref.fill(hilbert); - } - - A.fill(hilbert); - s.end(); - - /* Right-hand side */ - matrix B_potrs(N, 1, NB, 1, false, "B"); - matrix Bref_potrs(N, 1, NB, 1, false, "Bref"); - - if (check_result) - { - auto rhs_vals = [] _CCCL_HOST_DEVICE(size_t row, size_t /*unused*/) { - return 1.0 * (row + 1); - }; - B_potrs.fill(rhs_vals); - Bref_potrs.fill(rhs_vals); - } - - // // Compute ||Bref|| - double Bref_nrm2 = 0.0; - double res_nrm2 = 0.0; - - if (check_result) - { - PDNRM2_HOST(&Bref_potrs, &Bref_nrm2); - } - - cudaEvent_t startEvent_pdpotrf, stopEvent_pdpotrf; - float milliseconds_pdpotrf = 0; - - // for (size_t row = 0; row < A.mt; row++) - // { - // for (size_t col = 0; col <= row; col++) - // { - // cuda_safe_call(cudaSetDevice(A.get_preferred_devid(row, col))); - // NOOP(A, row, col); - // } - // } - - cuda_safe_call(cudaStreamSynchronize(ctx.fence())); - - cuda_safe_call(cudaEventCreate(&startEvent_pdpotrf)); - cuda_safe_call(cudaEventCreate(&stopEvent_pdpotrf)); - - cuda_safe_call(cudaEventRecord(startEvent_pdpotrf, ctx.fence())); - - PDPOTRF(A); - - cuda_safe_call(cudaEventRecord(stopEvent_pdpotrf, ctx.fence())); - - /* - * POTRS - */ - - if (check_result) - { - // Solve AX = B and put the result in B - PDPOTRS(A, B_potrs, CUBLAS_FILL_MODE_LOWER); - - // Compute (AX - B) - // Bref = (Aref*B - Bref) - PDGEMM(CUBLAS_OP_N, CUBLAS_OP_N, 1.0, Aref, B_potrs, -1.0, Bref_potrs); - - // Compute ||AX - B|| = ||Bref|| - PDNRM2_HOST(&Bref_potrs, &res_nrm2); - } - - ctx.finalize(); - - cuda_safe_call(cudaEventElapsedTime(&milliseconds_pdpotrf, startEvent_pdpotrf, stopEvent_pdpotrf)); - - double gflops_pdpotrf = 1.0 / 3.0 * ((double) N * (double) N * (double) N) / (1000000000.0); - std::cout << "[PDPOTRF] ELAPSED: " << milliseconds_pdpotrf - << " ms, GFLOPS: " << gflops_pdpotrf / (milliseconds_pdpotrf / 1000.0) << '\n'; - - if (check_result) - { - if (double residual = sqrt(res_nrm2) / sqrt(Bref_nrm2); residual >= 0.01) - { - std::cerr << "[POTRS] ||AX - B|| : " << sqrt(res_nrm2) << '\n'; - std::cerr << "[POTRS] ||B|| : " << sqrt(Bref_nrm2) << '\n'; - std::cerr << "[POTRS] RESIDUAL (||AX - B||/||B||) : " << residual << '\n'; - assert(!"Algorithm did not converge."); - } - } -} diff --git a/cccl_upstream/cudax/examples/stf/linear_algebra/07-potri.cu b/cccl_upstream/cudax/examples/stf/linear_algebra/07-potri.cu deleted file mode 100644 index affcb2a7..00000000 --- a/cccl_upstream/cudax/examples/stf/linear_algebra/07-potri.cu +++ /dev/null @@ -1,1678 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief This example implements the POTRI matrix inversion algorithm over multiple devices - * - * - */ -#include - -#include - -#include - -#include - -#define TILED - -using namespace cuda::experimental::stf; - -stream_ctx ctx; - -static std::unordered_map> cublas_handles; -static std::unordered_map> cusolver_handles; - -/* Get a CUBLAS handle valid on the current execution place, or initialize it lazily */ -cublasHandle_t& get_cublas_handle(const exec_place& ep = exec_place::current_device()) -{ - auto& result = cublas_handles[ep]; - if (result == cublasHandle_t()) - { // not found, default value inserted - // Lazy initialization, and save the handle for future use - cuda_try(cublasCreate(&result)); - } - return result; -} - -/* Get a CUSOLVER handle valid on the current execution place, or initialize it lazily */ -cusolverDnHandle_t& get_cusolver_handle(const exec_place& ep = exec_place::current_device()) -{ - auto& result = cusolver_handles[ep]; - if (result == cusolverDnHandle_t()) - { // not found, default value inserted - // Lazy initialization, and save the handle for future use - cuda_try(cusolverDnCreate(&result)); - } - return result; -} - -template -class matrix -{ -public: - matrix(int NROWS, int NCOLS, int BLOCKSIZE_ROWS, int BLOCKSIZE_COLS, bool is_sym, const char* _symbol = "matrix") - { - symbol = _symbol; - - sym_matrix = is_sym; - - m = NROWS; - mb = BLOCKSIZE_ROWS; - - n = NCOLS; - nb = BLOCKSIZE_COLS; - - assert(m % mb == 0); - assert(n % nb == 0); - - size_t s = m * n * sizeof(T); - // cuda_try(cudaMallocHost(&h_array, m*n*sizeof(T))); - // fprintf(stderr, "Allocating %ld x %ld x %ld = %ld bytes (%f GB) on host for %s\n", m, n, sizeof(T), s, - // s / (1024.0 * 1024.0 * 1024.0), _symbol); - h_array = (T*) malloc(s); - assert(h_array); - cuda_try(cudaHostRegister(h_array, s, cudaHostRegisterPortable)); - // cuda_try(cudaMalloc(&d_array, m*n*sizeof(T))); - - // Compute the number of blocks - mt = m / mb; - nt = n / nb; - - handles.resize(mt * nt); - - for (size_t colb = 0; colb < nt; colb++) - { - size_t low_rowb = sym_matrix ? colb : 0; - for (size_t rowb = low_rowb; rowb < mt; rowb++) - { - T* addr_h = get_block_h(rowb, colb); - auto& h = get_handle(rowb, colb); - -#ifdef TILED - // tiles are stored contiguously - size_t ld = mb; -#else - size_t ld = m; -#endif - std::ignore = ld; // work around compiler bug - h = ctx.logical_data(make_slice(addr_h, std::tuple{mb, nb}, ld)); - h.set_symbol(std::string(symbol) + "_" + std::to_string(rowb) + "_" + std::to_string(colb)); - h.set_write_back(false); - } - } - - cuda_try(cudaGetDeviceCount(&ndevs)); - for (int a = 1; a * a <= ndevs; a++) - { - if (ndevs % a == 0) - { - grid_p = a; - grid_q = ndevs / a; - } - } - - assert(grid_p * grid_q == ndevs); - - // std::cout << "FOUND " << ndevs << " DEVICES " - // << "p=" << grid_p << " q=" << grid_q << '\n'; - } - - int get_preferred_devid(int row, int col) - { - return (row % grid_p) + (col % grid_q) * grid_p; - } - - auto& get_handle(int row, int col) - { - return handles[row + col * mt]; - } - - size_t get_index(size_t row, size_t col) - { -#ifdef TILED - // Find which tile contains this element - int tile_row = row / mb; - int tile_col = col / nb; - - size_t tile_size = mb * nb; - - // Look for the index of the beginning of the tile - size_t tile_start = (tile_row + mt * tile_col) * tile_size; - - // Offset within the tile - size_t offset = (row % mb) + (col % nb) * mb; - - return tile_start + offset; -#else - return row + col * m; -#endif - } - - T* get_block_h(int brow, int bcol) - { - size_t index = get_index(brow * mb, bcol * nb); - return &h_array[index]; - } - - // Fill with func(Matrix*,row, col) - template - void fill(Fun&& fun) - { - nvtx_range r("fill"); - - // Fill blocks by blocks - for (size_t colb = 0; colb < nt; colb++) - { - size_t low_rowb = sym_matrix ? colb : 0; - for (size_t rowb = low_rowb; rowb < mt; rowb++) - { - // Each task fills a block - auto& h = get_handle(rowb, colb); - int devid = get_preferred_devid(rowb, colb); - - ctx.parallel_for(exec_place::device(devid), h.shape(), h.write()).set_symbol("INIT")->* - [=] _CCCL_DEVICE(size_t lrow, size_t lcol, auto sA) { - size_t row = lrow + rowb * sA.extent(0); - size_t col = lcol + colb * sA.extent(1); - sA(lrow, lcol) = fun(row, col); - }; - } - } - } - - // Print blocks - void print() - { - // print blocks by blocks - for (size_t colb = 0; colb < nt; colb++) - { - int low_rowb = sym_matrix ? colb : 0; - for (size_t rowb = low_rowb; rowb < mt; rowb++) - { - // Each task fills a block - ctx.host_launch(get_handle(rowb, colb).read())->*[=](auto sA) { - for (size_t lcol = 0; lcol < sA.extent(1); lcol++) - { - size_t col = lcol + colb * sA.extent(1); - for (size_t lrow = 0; lrow < sA.extent(0); lrow++) - { - size_t row = lrow + rowb * sA.extent(0); - - fprintf(stderr, "%d,%d : %le\n", row, col, sA(lrow, lcol)); - } - } - }; - } - } - } - - T* h_array; - T* d_array; - size_t m; // nrows - size_t n; // ncols - - // Is this a sym matrix ? (lower assumed) - bool sym_matrix; - - size_t mb; // block size (rows) - size_t nb; // block size (cols) - - size_t mt; // number of column blocks - size_t nt; // number of row blocks - - // abstract data handles - std::vector>> handles; - - const char* symbol; - - // for the mapping - int ndevs; - int grid_p, grid_q; -}; - -void DPOTRF(cublasFillMode_t uplo, matrix& A, int A_row, int A_col) -{ - auto& Akk = A.get_handle(A_row, A_col); - size_t m_akk = Akk.shape().extent(0); - // Note that the handle may be different from the actual handle... - int Lwork_expected; - cuda_safe_call(cusolverDnDpotrf_bufferSize(get_cusolver_handle(), uplo, m_akk, nullptr, 0, &Lwork_expected)); - - auto potrf_buffer = ctx.logical_data(shape_of>(Lwork_expected)); - potrf_buffer.set_allocator(ctx.get_default_allocator()); - - auto devInfo = ctx.logical_data(shape_of>(1)); - - auto t = ctx.task(Akk.rw(), potrf_buffer.write(), devInfo.write()); - t.set_symbol("DPOTRF"); - t->*[&](auto s, auto sAkk, auto buffer, auto info) { - auto& h = get_cusolver_handle(); - cuda_try(cusolverDnSetStream(h, s)); - - cuda_try(cusolverDnDpotrf( - h, - uplo, - sAkk.extent(0), - sAkk.data_handle(), - sAkk.stride(1), - buffer.data_handle(), - buffer.extent(0), - info.data_handle())); - }; -} - -void DTRTRI(cublasFillMode_t uplo, cublasDiagType_t diag, matrix& A, int A_row, int A_col) -{ - // Preallocate a buffer used by CUSOLVER - size_t workspaceInBytesOnDevice, workspaceInBytesOnHost; - int64_t m_a00 = A.mb; - assert(A.mb == A.nb); - - cuda_try(cusolverDnXtrtri_bufferSize( - get_cusolver_handle(), - uplo, - diag, - m_a00, - CUDA_R_64F /* DTRTRI */, - nullptr, - m_a00, - &workspaceInBytesOnDevice, - &workspaceInBytesOnHost)); - - // We don't support allocating buffers of 0 bytes ... XXX - if (workspaceInBytesOnHost == 0) - { - workspaceInBytesOnHost = 8; - } - - auto d_buffer = ctx.logical_data(shape_of>(workspaceInBytesOnDevice)); - auto h_buffer = ctx.logical_data(shape_of>(workspaceInBytesOnHost)); - d_buffer.set_allocator(ctx.get_default_allocator()); - h_buffer.set_allocator(ctx.get_default_allocator()); - - auto devInfo = ctx.logical_data(shape_of>(1)); - - auto t = - ctx.task(A.get_handle(A_row, A_col).rw(), d_buffer.write(), h_buffer.write(data_place::managed()), devInfo.write()); - t.set_symbol("DTRTRI"); - t->*[&](auto s, auto sA, auto dbuffer, auto hbuffer, auto info) { - auto& h = get_cusolver_handle(); - cuda_try(cusolverDnSetStream(h, s)); - - // DTRTRI(...) - cuda_try(cusolverDnXtrtri( - h, - uplo, - diag, - sA.extent(0), - CUDA_R_64F /* DTRTRI */, - sA.data_handle(), - sA.stride(1), - (double*) dbuffer.data_handle(), - workspaceInBytesOnDevice, - (double*) hbuffer.data_handle(), - workspaceInBytesOnHost, - info.data_handle())); - }; -} - -/* - * Note: this code was taken from CUSOLVER - * - * SLACPY copies all or part of a two-dimensional matrix A to another matrix B. - * - * up up_and_lo - * 1 0 upper triangle, including diagonal - * 0 0 lower triangle, including diagonal - * ? 1 whole matrix - * - * configuration: - * dim3 grids( m/VEC, m/BY ) - * dim3 threads(VEC,BY) - */ -template -__global__ void __launch_bounds__(1 << (VEC_LOG + BY_LOG)) - lacpy_kernel(int m, int n, const T_ELEM_SRC* A, size_t lda, T_ELEM_DST* B, size_t ldb, int up, int up_and_lo) -{ - const int VEC = (1 << VEC_LOG); - const int BY = (1 << BY_LOG); - - const int inx = threadIdx.x; - const int iny = threadIdx.y; - - const int ibx = blockIdx.x * VEC; - const int iby = blockIdx.y * BY; - - const int i = ibx + inx; - const int j = iby + iny; - - if (ibx >= m) - { - return; - } - if (iby >= n) - { - return; - } - - T_ELEM_SRC Areg = T_ELEM_SRC(0); - - if (up_and_lo) - { - /* - * copy whole matrix - DO 60 J = 1, N - DO 50 I = 1, M - B( I, J ) = A( I, J ) - 50 CONTINUE - 60 CONTINUE - */ - if ((i < m) && (j < n)) - { - Areg = A[i + j * lda]; - B[i + j * ldb] = T_ELEM_DST(Areg); - } - return; - } - - // only lower or upper triangle is copied. - if (up) - { - /* - * copy upper triangle, including diagonal - DO 20 J = 1, N - DO 10 I = 1, MIN( J, M ) - B( I, J ) = A( I, J ) - 10 CONTINUE - 20 CONTINUE - */ - if ((i <= min(j, m - 1)) && (j < n)) - { - Areg = A[i + j * lda]; - B[i + j * ldb] = T_ELEM_DST(Areg); - } - } - else - { - /* - * copy lower triangle, including diagonal - DO 40 J = 1, N - DO 30 I = J, M - B( I, J ) = A( I, J ) - 30 CONTINUE - 40 CONTINUE - - */ - if (((j <= i) && (i < m)) && (j < n)) - { - Areg = A[i + j * lda]; - B[i + j * ldb] = T_ELEM_DST(Areg); - } - } -} - -/* - * SLACPY copies all or part of a two-dimensional matrix A to another - * matrix B. - * - * Input - * ------- - * UPLO is CHARACTER*1 - * Specifies the part of the matrix A to be copied to B. - * = 'U': Upper triangular part - * = 'L': Lower triangular part - * Otherwise: All of the matrix A - * - * M is INTEGER - * The number of rows of the matrix A. - * M >= 0. - * - * N is INTEGER - * The number of columns of the matrix A. - * N >= 0. - * - * A is REAL array, dimension (LDA,N) - * The m by n matrix A. If UPLO = 'U', only the upper triangle - * or trapezoid is accessed; if UPLO = 'L', only the lower - * triangle or trapezoid is accessed. - * - * LDA is INTEGER - * The first dimension of the array A. LDA >= max(1,M). - * - * B is REAL array, dimension (LDB,N) - * On exit, B = A in the locations specified by UPLO. - * - * LDB is INTEGER - * The leading dimension of the array B. LDB >= max(1,M). - * - */ -template -cusolverStatus_t cusolverDnXlacpy( - cublasFillMode_t uplo, // "UPPER", B = upper(A) - // "LOWER", B = lower(A) - // otherwise, B = A - int m, - int n, - const T_ELEM_SRC* A, - int lda, - T_ELEM_DST* B, - int ldb, - cudaStream_t stream) -{ - cusolverStatus_t status = CUSOLVER_STATUS_SUCCESS; - cudaError_t cudaStat1 = cudaSuccess; - - int up = 0; - int up_and_lo = 0; - - // Quick return if possible - if ((0 >= m) || (0 >= n)) - { - return status; - } - - /* - * up up_and_lo - * 1 0 upper triangle, including diagonal - * 0 0 lower triangle, including diagonal - * ? 1 whole matrix - */ - if (CUBLAS_FILL_MODE_LOWER == uplo) - { - // Lower triangular part - up = 0; - } - else if (CUBLAS_FILL_MODE_UPPER == uplo) - { - // upper triangular part - up = 1; - } - else - { - up_and_lo = 1; // Otherwise: All of the matrix A - } - - const int VEC_LOG = 5; - const int BY_LOG = 3; - const int VEC = (1 << VEC_LOG); - const int BY = (1 << BY_LOG); - dim3 grids((m + VEC - 1) / VEC, (n + BY - 1) / BY); - dim3 threads(VEC, BY); - - lacpy_kernel - <<>>(m, n, A, (size_t) lda, B, (size_t) ldb, up, up_and_lo); - - cudaStat1 = cudaGetLastError(); /* launch error */ - if (cudaSuccess != cudaStat1) - { - fprintf(stderr, "Error (lacpy): %d\n", cudaStat1); - status = CUSOLVER_STATUS_EXECUTION_FAILED; - } - - return status; -} - -cusolverStatus_t cusolverDnDlacpy( - cublasFillMode_t uplo, // "UPPER", B = upper(A) - // "LOWER", B = lower(A) - // otherwise, B = A - int m, - int n, - const double* A, - int lda, - double* B, - int ldb, - cudaStream_t stream) -{ - return cusolverDnXlacpy(uplo, m, n, A, lda, B, ldb, stream); -} - -// Pretend there is a CUBLAS interface for DLAAUM -void cublasDnDlaaum_bufferSize(cublasHandle_t /*unused*/, int m, int n, size_t* Workspace_size) -{ - assert(Workspace_size); - *Workspace_size = m * n * sizeof(double); -} - -// Pretend there is a CUBLAS interface for DLAAUM -// A triangular -// Lower : A = A^T * A -// Upper : A = A A^T -void cublasDnDlaaum( - cublasHandle_t cublas_handle, - cublasFillMode_t uplo, - int m, - int n, - double* A, - int ldA, - double* Workspace_d, - size_t Workspace_size) -{ - cudaStream_t stream; - cuda_safe_call(cublasGetStream(cublas_handle, &stream)); - - // "Hand coded" - // We use a full copy of A ! - // fprintf(stderr, "GOT Workspace_size %ld ... expected %d\n", Workspace_size, m * n * sizeof(double)); - std::ignore = Workspace_size; - assert(Workspace_size >= m * n * sizeof(double)); - - double* B = Workspace_d; - int ldB = m; - - // Blank the buffer - cuda_safe_call(cudaMemsetAsync(B, 0, m * n * sizeof(double), stream)); - - // Copy A (with upper or lower 0 untouched) - cusolverDnDlacpy(uplo, m, n, A, ldA, B, ldB, stream); - - cublasDiagType_t diag = CUBLAS_DIAG_NON_UNIT; - const double one = 1.0; - - auto side = (uplo == CUBLAS_FILL_MODE_LOWER) ? CUBLAS_SIDE_LEFT : CUBLAS_SIDE_RIGHT; - - // LOWER: TRMM(A,B) : B = op(A) * B = A^T * B with A triangular (B = C in CUBLAS), CUBLAS_OP_T, CUBLAS_SIDE_RIGHT - // UPPER: TRMM(A,B) : B = B * op(A) = B A^T with A triangular (B = C in CUBLAS), CUBLAS_OP_T, CUBLAS_SIDE_RIGHT - cuda_safe_call(cublasDtrmm(cublas_handle, side, uplo, CUBLAS_OP_T, diag, m, n, &one, A, ldA, B, ldB, B, ldB)); - - // Copy B=AA^T back into A (with upper or lower 0 untouched) - cusolverDnDlacpy(uplo, m, n, B, ldB, A, ldA, stream); -} - -void DLAAUM(cublasFillMode_t uplo, matrix& A, int A_row, int A_col) -{ - int NB = A.mb; - size_t Lwork; - cublasDnDlaaum_bufferSize(get_cublas_handle(), NB, NB, &Lwork); - - auto d_buffer = ctx.logical_data(shape_of>(Lwork)); - - auto t = ctx.task(A.get_handle(A_row, A_col).rw(), d_buffer.write()); - t.set_symbol("DLAAUM"); - t->*[&](auto s, auto sA, auto buffer) { - auto& h = get_cublas_handle(); - cuda_try(cublasSetStream(h, s)); - - cublasDnDlaaum( - h, uplo, sA.extent(0), sA.extent(1), sA.data_handle(), sA.stride(1), (double*) buffer.data_handle(), Lwork); - }; -} - -void DGEMM( - cublasOperation_t transa, - cublasOperation_t transb, - double alpha, - matrix& A, - int A_row, - int A_col, - matrix& B, - int B_row, - int B_col, - double beta, - matrix& C, - int C_row, - int C_col) -{ - auto ignored = get_cublas_handle(); - auto t = - ctx.task(A.get_handle(A_row, A_col).read(), B.get_handle(B_row, B_col).read(), C.get_handle(C_row, C_col).rw()); - t.set_symbol("DGEMM"); - t->*[&](auto s, auto sA, auto sB, auto sC) { - auto& h = get_cublas_handle(); - cuda_try(cublasSetStream(h, s)); - - int k = (transa == CUBLAS_OP_N) ? sA.extent(1) : sA.extent(0); - cuda_try(cublasDgemm( - h, - transa, - transb, - sC.extent(0), - sC.extent(1), - k, - &alpha, - sA.data_handle(), - sA.stride(1), - sB.data_handle(), - sB.stride(1), - &beta, - sC.data_handle(), - sC.stride(1))); - }; -} - -void DSYMM( - cublasSideMode_t side, - cublasFillMode_t uplo, - double alpha, - matrix& A, - int A_row, - int A_col, - matrix& B, - int B_row, - int B_col, - double beta, - matrix& C, - int C_row, - int C_col) -{ - auto ignored = get_cublas_handle(); - auto t = - ctx.task(A.get_handle(A_row, A_col).read(), B.get_handle(B_row, B_col).read(), C.get_handle(C_row, C_col).rw()); - t.set_symbol("DSYMM"); - t->*[&](auto s, auto sA, auto sB, auto sC) { - auto& h = get_cublas_handle(); - cuda_try(cublasSetStream(h, s)); - - cuda_try(cublasDsymm( - h, - side, - uplo, - sC.extent(0), - sC.extent(1), - &alpha, - sA.data_handle(), - sA.stride(1), - sB.data_handle(), - sB.stride(1), - &beta, - sC.data_handle(), - sC.stride(1))); - }; -} - -void DSYRK( - cublasFillMode_t uplo, - cublasOperation_t trans, - double alpha, - matrix& A, - int A_row, - int A_col, - double beta, - matrix& C, - int C_row, - int C_col) -{ - auto ignored = get_cublas_handle(); - auto t = ctx.task(A.get_handle(A_row, A_col).read(), C.get_handle(C_row, C_col).rw()); - t.set_symbol("DSYRK"); - t->*[&](auto s, auto sA, auto sC) { - auto& h = get_cublas_handle(); - cuda_try(cublasSetStream(h, s)); - - // number of rows of matrix op(A) and C - int n = sC.extent(0); - - // number of columns of matrix op(A) - int k = (trans == CUBLAS_OP_N) ? sA.extent(1) : sA.extent(0); - - cuda_try( - cublasDsyrk(h, uplo, trans, n, k, &alpha, sA.data_handle(), sA.stride(1), &beta, sC.data_handle(), sC.stride(1))); - }; -} - -void DTRSM( - cublasSideMode_t side, - cublasFillMode_t uplo, - cublasOperation_t transa, - cublasDiagType_t diag, - double alpha, - matrix& A, - int A_row, - int A_col, - matrix& B, - int B_row, - int B_col) -{ - auto ignored = get_cublas_handle(); - auto t = ctx.task(A.get_handle(A_row, A_col).read(), B.get_handle(B_row, B_col).rw()); - t.set_symbol("DTRSM"); - t->*[&](auto s, auto sA, auto sB) { - auto& h = get_cublas_handle(); - cuda_try(cublasSetStream(h, s)); - - cuda_try(cublasDtrsm( - h, - side, - uplo, - transa, - diag, - sB.extent(0), - sB.extent(1), - &alpha, - sA.data_handle(), - sA.stride(1), - sB.data_handle(), - sB.stride(1))); - }; -} - -void DTRMM( - cublasSideMode_t side, - cublasFillMode_t uplo, - cublasOperation_t transa, - cublasDiagType_t diag, - double alpha, - matrix& A, - int A_row, - int A_col, - matrix& B, - int B_row, - int B_col) -{ - auto ignored = get_cublas_handle(); - auto t = ctx.task(A.get_handle(A_row, A_col).read(), B.get_handle(B_row, B_col).rw()); - t.set_symbol("DTRMM"); - t->*[&](auto s, auto sA, auto sB) { - auto& h = get_cublas_handle(); - cuda_try(cublasSetStream(h, s)); - - // Note : CUBLAS DTRMM implementation is out of place but supports in place by using the same buffer B and C - cuda_try(cublasDtrmm( - get_cublas_handle(), - side, - uplo, - transa, - diag, - sB.extent(0), - sB.extent(1), - &alpha, - sA.data_handle(), - sA.stride(1), - sB.data_handle(), - sB.stride(1), - sB.data_handle(), - sB.stride(1) /* same as B*/)); - }; -} - -void PDNRM2_HOST(matrix* A, double* result) -{ -#ifdef HAVE_DOT - ctx.get_dot()->set_current_color("red"); -#endif - - for (size_t rowb = 0; rowb < A->mt; rowb++) - { - for (size_t colb = 0; colb < A->nt; colb++) - { - ctx.host_launch(A->get_handle(rowb, colb).read())->*[=](auto sA) { - double res2 = 0.0; - for (size_t col = 0; col < sA.extent(1); col++) - { - for (size_t row = 0; row < sA.extent(0); row++) - { - double v = sA(row, col); - res2 += v * v; - } - } - *result += res2; - }; - } - } -} - -void PDPOTRF(matrix& A) -{ - nvtx_range r("PDPOTRF"); - -#ifdef HAVE_DOT - ctx.get_dot()->set_current_color("yellow"); -#endif - - assert(A.m == A.n); - assert(A.mt == A.nt); - - int NBLOCKS = A.mt; - assert(A.mb == A.nb); - - for (int K = 0; K < NBLOCKS; K++) - { - cuda_try(cudaSetDevice(A.get_preferred_devid(K, K))); - DPOTRF(CUBLAS_FILL_MODE_LOWER, A, K, K); - - for (int row = K + 1; row < NBLOCKS; row++) - { - cuda_try(cudaSetDevice(A.get_preferred_devid(row, K))); - DTRSM(CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_LOWER, CUBLAS_OP_T, CUBLAS_DIAG_NON_UNIT, 1.0, A, K, K, A, row, K); - - for (int col = K + 1; col < row; col++) - { - cuda_try(cudaSetDevice(A.get_preferred_devid(row, col))); - DGEMM(CUBLAS_OP_N, CUBLAS_OP_T, -1.0, A, row, K, A, col, K, 1.0, A, row, col); - } - - cuda_try(cudaSetDevice(A.get_preferred_devid(row, row))); - DSYRK(CUBLAS_FILL_MODE_LOWER, CUBLAS_OP_N, -1.0, A, row, K, 1.0, A, row, row); - } - } -} - -// Algorithm from PLASMA -void PDTRSM(cublasSideMode_t side, - cublasFillMode_t uplo, - cublasOperation_t trans, - cublasDiagType_t diag, - double alpha, - matrix& A, - matrix& B) -{ - nvtx_range r("PDTRSM"); - // std::cout << "[PDTRSM] START B MT " << B.mt << " NT " << B.nt << '\n'; - - if (side == CUBLAS_SIDE_LEFT) - { - if (uplo == CUBLAS_FILL_MODE_UPPER) - { - // TODO - abort(); - } - else - { - //=========================================== - // CUBLAS_SIDE_LEFT / CUBLAS_FILL_MODE_LOWER / CUBLAS_OP_N - //=========================================== - if (trans == CUBLAS_OP_N) - { - for (size_t k = 0; k < B.mt; k++) - { - double lalpha = k == 0 ? alpha : 1.0; - for (size_t n = 0; n < B.nt; n++) - { - cuda_try(cudaSetDevice(A.get_preferred_devid(k, k))); - DTRSM(side, uplo, trans, diag, lalpha, A, k, k, B, k, n); - } - for (size_t m = k + 1; m < B.mt; m++) - { - for (size_t n = 0; n < B.nt; n++) - { - cuda_try(cudaSetDevice(A.get_preferred_devid(m, k))); - DGEMM(CUBLAS_OP_N, CUBLAS_OP_N, -1.0, A, m, k, B, k, n, lalpha, B, m, n); - } - } - } - } - //================================================ - // CUBLAS_SIDE_LEFT / CUBLAS_FILL_MODE_LOWER / CUBLAS_OP_[C|T] - //================================================ - else - { - for (size_t k = 0; k < B.mt; k++) - { - double lalpha = k == 0 ? alpha : 1.0; - for (size_t n = 0; n < B.nt; n++) - { - cuda_try(cudaSetDevice(A.get_preferred_devid(B.mt - k - 1, B.mt - k - 1))); - DTRSM(side, uplo, trans, diag, lalpha, A, B.mt - k - 1, B.mt - k - 1, B, B.mt - k - 1, n); - } - for (size_t m = k + 1; m < B.mt; m++) - { - for (size_t n = 0; n < B.nt; n++) - { - cuda_try(cudaSetDevice(A.get_preferred_devid(B.mt - k - 1, B.mt - 1 - m))); - DGEMM( - trans, CUBLAS_OP_N, -1.0, A, B.mt - k - 1, B.mt - 1 - m, B, B.mt - k - 1, n, lalpha, B, B.mt - 1 - m, n); - } - } - } - } - } - } - else - { - // TODO - abort(); - } - // std::cout << "[PDTRSM] END" << '\n'; -} - -void PDPOTRS(matrix& A, matrix& B, cublasFillMode_t uplo) -{ - nvtx_range r("PDPOTRS"); - -#ifdef HAVE_DOT - ctx.get_dot()->set_current_color("green"); -#endif - - // std::cout << "[PDPOTRS] START" << '\n'; - // Call the parallel functions. - PDTRSM( - CUBLAS_SIDE_LEFT, uplo, uplo == CUBLAS_FILL_MODE_UPPER ? CUBLAS_OP_T : CUBLAS_OP_N, CUBLAS_DIAG_NON_UNIT, 1.0, A, B); - -#ifdef HAVE_DOT - ctx.get_dot()->set_current_color("darkgreen"); -#endif - - PDTRSM( - CUBLAS_SIDE_LEFT, uplo, uplo == CUBLAS_FILL_MODE_UPPER ? CUBLAS_OP_N : CUBLAS_OP_T, CUBLAS_DIAG_NON_UNIT, 1.0, A, B); - // std::cout << "[PDPOTRS] END" << '\n'; -} - -/***************************************************************************/ /** - * Parallel tile matrix-matrix - *multiplication. - * @see plasma_omp_dgemm - ******************************************************************************/ -void PDGEMM(cublasOperation_t transa, - cublasOperation_t transb, - double alpha, - matrix& A, - matrix& B, - double beta, - matrix& C) -{ - nvtx_range r("PDGEMM"); - -#ifdef HAVE_DOT - reserved::dot::set_current_color("blue"); -#endif - - for (size_t m = 0; m < C.mt; m++) - { - for (size_t n = 0; n < C.nt; n++) - { - cuda_try(cudaSetDevice(C.get_preferred_devid(m, n))); - - //========================================= - // alpha*A*B does not contribute; scale C - //========================================= - int inner_k = transa == CUBLAS_OP_N ? A.n : A.m; - if (alpha == 0.0 || inner_k == 0) - { - DGEMM(transa, transb, alpha, A, 0, 0, B, 0, 0, beta, C, m, n); - } - else if (transa == CUBLAS_OP_N) - { - //================================ - // CUBLAS_OP_N / CUBLAS_OP_N - //================================ - if (transb == CUBLAS_OP_N) - { - assert(A.nt == B.mt); - for (size_t k = 0; k < A.nt; k++) - { - double zbeta = k == 0 ? beta : 1.0; - DGEMM(transa, transb, alpha, A, m, k, B, k, n, zbeta, C, m, n); - } - } - //===================================== - // CUBLAS_OP_N / CUBLAS_OP_T - //===================================== - else - { - for (size_t k = 0; k < A.nt; k++) - { - double zbeta = k == 0 ? beta : 1.0; - DGEMM(transa, transb, alpha, A, m, k, B, n, k, zbeta, C, m, n); - } - } - } - else - { - //===================================== - // CUBLAS_OP_T / CUBLAS_OP_N - //===================================== - if (transb == CUBLAS_OP_N) - { - for (size_t k = 0; k < A.mt; k++) - { - double zbeta = k == 0 ? beta : 1.0; - DGEMM(transa, transb, alpha, A, k, m, B, k, n, zbeta, C, m, n); - } - } - //========================================== - // CUBLAS_OP_T / CUBLAS_OP_T - //========================================== - else - { - for (size_t k = 0; k < A.mt; k++) - { - double zbeta = k == 0 ? beta : 1.0; - DGEMM(transa, transb, alpha, A, k, m, B, n, k, zbeta, C, m, n); - } - } - } - } - } -} - -/* - * Algorithm taken from the PLASMA library - */ -// We assume a lower triangular matrix (uplo == CUBLAS_FILL_MODE_LOWER) -void PDTRTRI(matrix& A, cublasFillMode_t uplo, cublasDiagType_t diag) -{ - nvtx_range r("PDTRTRI"); - - assert(uplo == CUBLAS_FILL_MODE_LOWER); - - for (size_t k = 0; k < A.nt; k++) - { - for (size_t m = k + 1; m < A.mt; m++) - { - cuda_try(cudaSetDevice(A.get_preferred_devid(m, k))); - DTRSM(CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_LOWER, CUBLAS_OP_N, diag, -1.0, A, k, k, A, m, k); - } - for (size_t m = k + 1; m < A.mt; m++) - { - for (size_t n = 0; n < k; n++) - { - cuda_try(cudaSetDevice(A.get_preferred_devid(m, n))); - DGEMM(CUBLAS_OP_N, CUBLAS_OP_N, 1.0, A, m, k, A, k, n, 1.0, A, m, n); - } - } - for (size_t n = 0; n < k; n++) - { - cuda_try(cudaSetDevice(A.get_preferred_devid(k, n))); - DTRSM(CUBLAS_SIDE_LEFT, CUBLAS_FILL_MODE_LOWER, CUBLAS_OP_N, diag, 1.0, A, k, k, A, k, n); - } - - // DTRTRI(...) - cuda_try(cudaSetDevice(A.get_preferred_devid(k, k))); - DTRTRI(uplo, diag, A, k, k); - } -} - -/* - * Algorithm taken from the PLASMA library - */ -// We assume a lower triangular matrix (uplo == CUBLAS_FILL_MODE_LOWER) -void PDLAUUM(matrix& A, cublasFillMode_t uplo) -{ - nvtx_range r("PDLAUUM"); - assert(uplo == CUBLAS_FILL_MODE_LOWER); - - for (size_t k = 0; k < A.mt; k++) - { - for (size_t n = 0; n < k; n++) - { - cuda_try(cudaSetDevice(A.get_preferred_devid(n, n))); - DSYRK(CUBLAS_FILL_MODE_LOWER, CUBLAS_OP_T, 1.0, A, k, n, 1.0, A, n, n); - - for (size_t m = n + 1; m < k; m++) - { - cuda_try(cudaSetDevice(A.get_preferred_devid(m, n))); - DGEMM(CUBLAS_OP_T, CUBLAS_OP_N, 1.0, A, k, m, A, k, n, 1.0, A, m, n); - } - } - for (size_t n = 0; n < k; n++) - { - cuda_try(cudaSetDevice(A.get_preferred_devid(k, n))); - DTRMM(CUBLAS_SIDE_LEFT, CUBLAS_FILL_MODE_LOWER, CUBLAS_OP_T, CUBLAS_DIAG_NON_UNIT, 1.0, A, k, k, A, k, n); - } - - // LAAUM (Akk RW) (compute Akk^T * Akk) - cuda_try(cudaSetDevice(A.get_preferred_devid(k, k))); - DLAAUM(uplo, A, k, k); - } -} - -void PDSYMM(cublasSideMode_t side, - cublasFillMode_t uplo, - double alpha, - matrix& A, - matrix& B, - double beta, - matrix& C) -{ - nvtx_range r("PDSYMM"); - - size_t k, m, n; - double zbeta; - double zone = (double) 1.0; - - for (m = 0; m < C.mt; m++) - { - for (n = 0; n < C.nt; n++) - { - cuda_try(cudaSetDevice(C.get_preferred_devid(m, n))); - /* - * CUBLAS_SIDE_LEFT / CUBLAS_FILL_MODE_LOWER - */ - if (side == CUBLAS_SIDE_LEFT) - { - if (uplo == CUBLAS_FILL_MODE_LOWER) - { - for (k = 0; k < C.mt; k++) - { - zbeta = k == 0 ? beta : zone; - if (k < m) - { - DGEMM(CUBLAS_OP_N, CUBLAS_OP_N, alpha, A, m, k, B, k, n, zbeta, C, m, n); - } - else - { - if (k == m) - { - DSYMM(side, uplo, alpha, A, k, k, B, k, n, zbeta, C, m, n); - } - else - { - DGEMM(CUBLAS_OP_T, CUBLAS_OP_N, alpha, A, k, m, B, k, n, zbeta, C, m, n); - } - } - } - } - /* - * CUBLAS_SIDE_LEFT / CUBLAS_FILL_MODE_UPPER - */ - else - { - for (k = 0; k < C.mt; k++) - { - zbeta = k == 0 ? beta : zone; - if (k < m) - { - DGEMM(CUBLAS_OP_T, CUBLAS_OP_N, alpha, A, k, m, B, k, n, zbeta, C, m, n); - } - else - { - if (k == m) - { - DSYMM(side, uplo, alpha, A, k, k, B, k, n, zbeta, C, m, n); - } - else - { - DGEMM(CUBLAS_OP_N, CUBLAS_OP_N, alpha, A, m, k, B, k, n, zbeta, C, m, n); - } - } - } - } - } - /* - * CUBLAS_SIDE_RIGHT / CUBLAS_FILL_MODE_LOWER - */ - else - { - if (uplo == CUBLAS_FILL_MODE_LOWER) - { - for (k = 0; k < C.nt; k++) - { - zbeta = k == 0 ? beta : zone; - if (k < n) - { - DGEMM(CUBLAS_OP_N, CUBLAS_OP_T, alpha, B, m, k, A, n, k, zbeta, C, m, n); - } - else - { - if (k == n) - { - DSYMM(side, uplo, alpha, A, k, k, B, m, k, zbeta, C, m, n); - } - else - { - DGEMM(CUBLAS_OP_N, CUBLAS_OP_N, alpha, B, m, k, A, k, n, zbeta, C, m, n); - } - } - } - } - /* - * CUBLAS_SIDE_RIGHT / CUBLAS_FILL_MODE_UPPER - */ - else - { - for (k = 0; k < C.nt; k++) - { - zbeta = k == 0 ? beta : zone; - if (k < n) - { - DGEMM(CUBLAS_OP_N, CUBLAS_OP_N, alpha, B, m, k, A, k, n, zbeta, C, m, n); - } - else - { - if (k == n) - { - DSYMM(side, uplo, alpha, A, k, k, B, m, k, zbeta, C, m, n); - } - else - { - DGEMM(CUBLAS_OP_N, CUBLAS_OP_T, alpha, B, m, k, A, n, k, zbeta, C, m, n); - } - } - } - } - } - } - } -} - -void PDTRMM(cublasSideMode_t side, - cublasFillMode_t uplo, - cublasOperation_t trans, - cublasDiagType_t diag, - double alpha, - matrix& A, - matrix& B) -{ - if (side == CUBLAS_SIDE_LEFT) - { - if (uplo == CUBLAS_FILL_MODE_UPPER) - { - //=========================================== - // CUBLAS_SIDE_LEFT / CUBLAS_FILL_MODE_UPPER / CUBLAS_OP_N - //=========================================== - if (trans == CUBLAS_OP_N) - { - for (size_t m = 0; m < B.mt; m++) - { - for (size_t n = 0; n < B.nt; n++) - { - cuda_try(cudaSetDevice(B.get_preferred_devid(m, n))); - - DTRMM(side, uplo, trans, diag, alpha, A, m, m, B, m, n); - - for (size_t k = m + 1; k < A.mt; k++) - { - DGEMM(trans, CUBLAS_OP_N, alpha, A, m, k, B, k, n, 1.0, B, m, n); - } - } - } - } - //================================================ - // CUBLAS_SIDE_LEFT / CUBLAS_FILL_MODE_UPPER / CUBLAS_OP_T - //================================================ - else - { - for (ssize_t m = B.mt - 1; m > -1; m--) - { - for (size_t n = 0; n < B.nt; n++) - { - cuda_try(cudaSetDevice(B.get_preferred_devid(m, n))); - - DTRMM(side, uplo, trans, diag, alpha, A, m, m, B, m, n); - - for (ssize_t k = 0; k < m; k++) - { - DGEMM(trans, CUBLAS_OP_N, alpha, A, k, m, B, k, n, 1.0, B, m, n); - } - } - } - } - } - else - { - //=========================================== - // CUBLAS_SIDE_LEFT / CUBLAS_FILL_MODE_LOWER / CUBLAS_OP_N - //=========================================== - if (trans == CUBLAS_OP_N) - { - for (ssize_t m = B.mt - 1; m > -1; m--) - { - for (size_t n = 0; n < B.nt; n++) - { - cuda_try(cudaSetDevice(B.get_preferred_devid(m, n))); - - DTRMM(side, uplo, trans, diag, alpha, A, m, m, B, m, n); - - for (ssize_t k = 0; k < m; k++) - { - DGEMM(trans, CUBLAS_OP_N, alpha, A, m, k, B, k, n, 1.0, B, m, n); - } - } - } - } - //================================================ - // CUBLAS_SIDE_LEFT / CUBLAS_FILL_MODE_LOWER / CUBLAS_OP_T - //================================================ - else - { - for (size_t m = 0; m < B.mt; m++) - { - for (size_t n = 0; n < B.nt; n++) - { - DTRMM(side, uplo, trans, diag, alpha, A, m, m, B, m, n); - - for (size_t k = m + 1; k < A.mt; k++) - { - DGEMM(trans, CUBLAS_OP_N, alpha, A, k, m, B, k, n, 1.0, B, m, n); - } - } - } - } - } - } - else - { - if (uplo == CUBLAS_FILL_MODE_UPPER) - { - //============================================ - // CUBLAS_SIDE_RIGHT / CUBLAS_FILL_MODE_UPPER / CUBLAS_OP_N - //============================================ - if (trans == CUBLAS_OP_N) - { - for (ssize_t n = B.nt - 1; n > -1; n--) - { - for (size_t m = 0; m < B.mt; m++) - { - cuda_try(cudaSetDevice(B.get_preferred_devid(m, n))); - - DTRMM(side, uplo, trans, diag, alpha, A, n, n, B, m, n); - - for (ssize_t k = 0; k < n; k++) - { - DGEMM(CUBLAS_OP_N, trans, alpha, B, m, k, A, k, n, 1.0, B, m, n); - } - } - } - } - //================================================= - // CUBLAS_SIDE_RIGHT / CUBLAS_FILL_MODE_UPPER / Plasma[_Conj]Trans - //================================================= - else - { - for (size_t n = 0; n < B.nt; n++) - { - for (size_t m = 0; m < B.mt; m++) - { - cuda_try(cudaSetDevice(B.get_preferred_devid(m, n))); - - DTRMM(side, uplo, trans, diag, alpha, A, n, n, B, m, n); - - for (size_t k = n + 1; k < A.mt; k++) - { - DGEMM(CUBLAS_OP_N, trans, alpha, B, m, k, A, n, k, 1.0, B, m, n); - } - } - } - } - } - else - { - //============================================ - // CUBLAS_SIDE_RIGHT / CUBLAS_FILL_MODE_LOWER / CUBLAS_OP_N - //============================================ - if (trans == CUBLAS_OP_N) - { - for (size_t n = 0; n < B.nt; n++) - { - for (size_t m = 0; m < B.mt; m++) - { - cuda_try(cudaSetDevice(B.get_preferred_devid(m, n))); - - DTRMM(side, uplo, trans, diag, alpha, A, n, n, B, m, n); - - for (size_t k = n + 1; k < A.mt; k++) - { - DGEMM(CUBLAS_OP_N, trans, alpha, B, m, k, A, k, n, 1.0, B, m, n); - } - } - } - } - //================================================= - // CUBLAS_SIDE_RIGHT / CUBLAS_FILL_MODE_LOWER / Plasma[_Conj]Trans - //================================================= - else - { - for (ssize_t n = B.nt - 1; n > -1; n--) - { - for (size_t m = 0; m < B.mt; m++) - { - cuda_try(cudaSetDevice(B.get_preferred_devid(m, n))); - - DTRMM(side, uplo, trans, diag, alpha, A, n, n, B, m, n); - - for (ssize_t k = 0; k < n; k++) - { - DGEMM(CUBLAS_OP_N, trans, alpha, B, m, k, A, n, k, 1.0, B, m, n); - } - } - } - } - } - } -} - -// Taken from Chameleon (INRIA) -// All the formula are reported in the LAPACK Lawn 41: -// http://www.netlib.org/lapack/lawns/lawn41.ps -#define FMULS_POTRI(__n) ((double) (__n) * ((2. / 3.) + (double) (__n) * ((1. / 3.) * (double) (__n) + 1.))) -#define FADDS_POTRI(__n) ((double) (__n) * ((1. / 6.) + (double) (__n) * ((1. / 3.) * (double) (__n) - 0.5))) -double flops_dpotri(double __n) -{ - double flops = (FMULS_POTRI((__n)) + FADDS_POTRI((__n))); - return flops; -} - -void run(int N, int NB) -{ - // Use pools of preallocated blocks - auto fixed_alloc = block_allocator(ctx, NB * NB * sizeof(double)); - ctx.set_allocator(fixed_alloc); - - // Set up CUBLAS and CUSOLVER - int ndevs; - cuda_try(cudaGetDeviceCount(&ndevs)); - - for (int d = 0; d < ndevs; d++) - { - auto ldummy = ctx.logical_data(shape_of>(1)); - ctx.task(exec_place::device(d), ldummy.write())->*[](cudaStream_t, auto) { - get_cublas_handle(); - get_cusolver_handle(); - }; - - ctx.task(exec_place::host(), ldummy.write(data_place::managed()))->*[](cudaStream_t, auto) {}; - } - - cuda_try(cudaSetDevice(0)); - - cudaStream_t timing_stream; - cuda_try(cudaStreamCreate(&timing_stream)); - - matrix A(N, N, NB, NB, true, "A"); - matrix Aref(N, N, NB, NB, false, "Aref"); - - // (Hilbert matrix + 2*N*Id) to have a diagonal dominant matrix - auto hilbert = [=] _CCCL_HOST_DEVICE(size_t row, size_t col) { - return 1.0 / (col + row + 1.0) + 2.0 * N * (col == row); - }; - - Aref.fill(hilbert); - A.fill(hilbert); - - /* Right-hand side */ - matrix B_potrs(N, 1, NB, 1, false, "B"); - matrix Bref_potrs(N, 1, NB, 1, false, "Bref"); - - auto rhs_vals = [] _CCCL_HOST_DEVICE(size_t row, size_t /*unused*/) { - return 1.0 * (row + 1); - }; - - B_potrs.fill(rhs_vals); - Bref_potrs.fill(rhs_vals); - - int check_result = 1; - if (getenv("CHECK_RESULT")) - { - check_result = atoi(getenv("CHECK_RESULT")); - } - - int check_result_potrs = check_result; - if (getenv("CHECK_RESULT_POTRS")) - { - check_result_potrs = atoi(getenv("CHECK_RESULT_POTRS")); - } - - // // Compute ||Bref|| - double Bref_nrm2 = 0.0; - double res_nrm2 = 0.0; - - if (check_result_potrs) - { - PDNRM2_HOST(&Bref_potrs, &Bref_nrm2); - } - - cudaEvent_t startEvent, stopEvent; - - cuda_safe_call(cudaSetDevice(0)); - cuda_safe_call(cudaStreamSynchronize(ctx.fence())); - cuda_safe_call(cudaEventCreate(&startEvent)); - cuda_safe_call(cudaEventCreate(&stopEvent)); - cuda_safe_call(cudaEventRecord(startEvent, ctx.fence())); - - ctx.get_dot()->set_current_color("green"); - PDPOTRF(A); - ctx.get_dot()->set_current_color("white"); - - /* - * POTRS - */ - - if (check_result_potrs) - { - // Solve AX = B and put the result in B - PDPOTRS(A, B_potrs, CUBLAS_FILL_MODE_LOWER); - - // Compute (AX - B) - // Bref = (Aref*B - Bref) - PDGEMM(CUBLAS_OP_N, CUBLAS_OP_N, 1.0, Aref, B_potrs, -1.0, Bref_potrs); - - // Compute ||AX - B|| = ||Bref|| - PDNRM2_HOST(&Bref_potrs, &res_nrm2); - } - - /* - * POTRI - */ - /* PDPOTRI = PDTRTRI + PDLAUUM */ - - // PDTRTRI : La^-1 (invert A) - // fprintf(stderr, "A=La before POTRI\n"); - // A.print(); - - ctx.get_dot()->set_current_color("yellow"); - PDTRTRI(A, CUBLAS_FILL_MODE_LOWER, CUBLAS_DIAG_NON_UNIT); - ctx.get_dot()->set_current_color("white"); - - // fprintf(stderr, "A=La^-1 after POTRI\n"); - // A.print(); - - // Computes the lower part of A^tA (La^-t La^-1) - ctx.get_dot()->set_current_color("blue"); - PDLAUUM(A, CUBLAS_FILL_MODE_LOWER); - ctx.get_dot()->set_current_color("white"); - - double b_nrm2_potri = 0.0; - double res_nrm2_potri = 0.0; - - if (check_result) - { - /* Right-hand side */ - matrix B_potri(N, 1, NB, 1, false, "B_potri"); - matrix Bref_potri(N, 1, NB, 1, false, "Bref_potri"); - - // auto rhs_vals = [](matrix& mat, int row, int col) { return 1.0 * (row + 1); }; - B_potri.fill(rhs_vals); - Bref_potri.fill(rhs_vals); - - // AX = B, X = A^-1 B - // LLt X = B, X = (LLt)^-1 B = L^-t L^-1 B - // Compute Bref_potri = (A^-1 B - B) - PDNRM2_HOST(&Bref_potri, &b_nrm2_potri); - - // B = (A^-1)*B (A triangular lower, B_potri full) - // fprintf(stderr, "B_potri before PDTRMM\n"); - // B_potri.print(); - // - // fprintf(stderr, "A before PDTRMM\n"); - // A.print(); - - // B_tmp = 0 (to avoid NaN*0.0) - matrix B_tmp(N, 1, NB, 1, false, "B_tmp"); - auto zero_vals = [] _CCCL_HOST_DEVICE(size_t /* unused */, size_t /*unused*/) { - return 0.0; - }; - B_tmp.fill(zero_vals); - - // B_tmp = A * B_potri + 0*B_tmp - PDSYMM(CUBLAS_SIDE_LEFT, CUBLAS_FILL_MODE_LOWER, 1.0, A, B_potri, 0.0, B_tmp); - - // fprintf(stderr, "B_potri after PDTRMM\n"); - // B_potri.print(); - - // res = A X - B - PDGEMM(CUBLAS_OP_N, CUBLAS_OP_N, 1.0, Aref, B_tmp, -1.0, Bref_potri); - - // fprintf(stderr, "Bref_potri after PDGEMM\n"); - // Bref_potri.print(); - - // Compute residual - PDNRM2_HOST(&Bref_potri, &res_nrm2_potri); - } - - cuda_safe_call(cudaSetDevice(0)); - cuda_safe_call(cudaEventRecord(stopEvent, ctx.fence())); - - ctx.finalize(); - - if (check_result_potrs) - { - double residual = sqrt(res_nrm2) / sqrt(Bref_nrm2); - // std::cout << "[POTRS] ||AX - B|| : " << sqrt(res_nrm2) << '\n'; - // std::cout << "[POTRS] ||B|| : " << sqrt(Bref_nrm2) << '\n'; - // std::cout << "[POTRS] RESIDUAL (||AX - B||/||B||) : " << residual << '\n'; - EXPECT(residual < 0.01); - } - - if (check_result) - { - double residual_potri = sqrt(res_nrm2_potri) / sqrt(b_nrm2_potri); - // std::cout << "[POTRI] RESIDUAL ||A * ((A^-1)B) - B|| : " << sqrt(res_nrm2_potri) << '\n'; - // std::cout << "[POTRI] RESIDUAL ||B|| : " << sqrt(b_nrm2_potri) << '\n'; - // std::cout << "[POTRI] RESIDUAL (||A * ((A^-1)B) - B||/||B||) : " << residual_potri << '\n'; - EXPECT(residual_potri < 0.0001); - } - - // // Compute Aref * A^-1 in Aref (A^-1 is lower triangular) - // PDTRMM(CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_LOWER, CUBLAS_OP_N, CUBLAS_DIAG_NON_UNIT, 1.0, A, Aref); - - // // This should be almost identity - // Aref.print(); - -#if 0 - - std::cout << "Print A^-1 after PDLAUUM : " << '\n'; - A.print(); - - std::cout << "RES after AX - B POTRI : " << '\n'; - Bref_potri.print(); - - // This should be almost identity - Aref.print(); -#endif - - float milliseconds; - cuda_safe_call(cudaEventElapsedTime(&milliseconds, startEvent, stopEvent)); - - double gflops = flops_dpotri((double) N) / (1000000000.0); - std::cout << "[PDPOTRI] ELAPSED: " << milliseconds << " ms, GFLOPS: " << gflops / (milliseconds / 1000.0) << '\n'; -} - -int main(int argc, char** argv) -{ - int N = 1024; - int NB = 128; - - if (argc > 1) - { - N = atoi(argv[1]); - } - - if (argc > 2) - { - NB = atoi(argv[2]); - } - - assert(N % NB == 0); - - run(N, NB); -} diff --git a/cccl_upstream/cudax/examples/stf/linear_algebra/burger.cu b/cccl_upstream/cudax/examples/stf/linear_algebra/burger.cu deleted file mode 100644 index 0eea4de6..00000000 --- a/cccl_upstream/cudax/examples/stf/linear_algebra/burger.cu +++ /dev/null @@ -1,372 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Sparse conjugate gradient algorithm - */ - -#include - -#include -#include -#include -#include -#include - -#include "dot.cuh" -#include "newton_solver.cuh" - -using namespace cuda::experimental::stf; - -#if !_CCCL_CTK_BELOW(12, 4) - -void build_full_csr_structure(size_t* row_offsets, size_t* col_indices, size_t N) -{ - size_t nnz = 0; - row_offsets[0] = 0; - - for (size_t row = 0; row < N; row++) - { - if (row == 0 || row == N - 1) - { - // Boundary rows: only diagonal entry (identity for BC: u[i] = prescribed_value) - col_indices[nnz++] = row; - } - else - { - // Interior rows: tridiagonal structure (left, center, right) - col_indices[nnz++] = row - 1; // left neighbor - col_indices[nnz++] = row; // center (diagonal) - col_indices[nnz++] = row + 1; // right neighbor - } - row_offsets[row + 1] = nnz; - } -} - -template -void assemble_jacobian_full( - ctx_t& ctx, vector_t U, vector_t values, size_t N, double h, double dt, double nu) -{ - ctx.parallel_for(box(N), U.read(), values.write()).set_symbol("assemble_jacobian_full") - ->*[N, h, dt, nu] __device__(size_t row, auto dU, auto dvalues) { - if (row == 0) - { - // Left boundary: u[0] = 0 (homogeneous Dirichlet) - // Jacobian row: [1, 0, 0, ..., 0] - size_t val_idx = 0; // First entry in CSR values array - dvalues[val_idx] = 1.0; - } - else if (row == N - 1) - { - // Right boundary: u[N-1] = 0 (homogeneous Dirichlet) - // Jacobian row: [0, ..., 0, 1] - size_t val_idx = 1 + 3 * (N - 2); // Last entry in CSR values array - dvalues[val_idx] = 1.0; - } - else - { - // Interior point: Burger's equation discretization - double u_i = dU[row]; - double u_ip1 = dU[row + 1]; - double u_im1 = dU[row - 1]; - - // Jacobian entries: ∂F_i/∂u_{i-1}, ∂F_i/∂u_i, ∂F_i/∂u_{i+1} - double left = -u_i / (2 * h) - nu / (h * h); - double center = 1.0 / dt + (u_ip1 - u_im1) / (2 * h) + 2.0 * nu / (h * h); - double right = u_i / (2 * h) - nu / (h * h); - - // CSR indexing for interior row i: starts at 1 + 3*(i-1) - size_t val_idx = 1 + 3 * (row - 1); - dvalues[val_idx] = left; // ∂F_i/∂u_{i-1} - dvalues[val_idx + 1] = center; // ∂F_i/∂u_i - dvalues[val_idx + 2] = right; // ∂F_i/∂u_{i+1} - } - }; -} - -// residual: length N (full system including boundaries) -template -void compute_residual_full( - ctx_t& ctx, vector_t U, vector_t U_prev, vector_t residual, size_t N, double h, double dt, double nu) -{ - ctx.parallel_for(box(N), residual.write(), U.read(), U_prev.read()).set_symbol("compute_residual_full") - ->*[N, h, dt, nu] __device__(size_t i, auto dresidual, auto dU, auto dU_prev) { - if (i == 0) - { - // Left boundary condition: u[0] = 0 - dresidual(i) = dU(i) - 0.0; - } - else if (i == N - 1) - { - // Right boundary condition: u[N-1] = 0 - dresidual(i) = dU(i) - 0.0; - } - else - { - // Interior point: Burger's equation F_i = ∂u/∂t + u*∂u/∂x - nu*∂²u/∂x² - double u_i = dU(i); - double u_ip1 = dU(i + 1); - double u_im1 = dU(i - 1); - - double term_time = (u_i - dU_prev(i)) / dt; // ∂u/∂t - double term_conv = u_i * (u_ip1 - u_im1) / (2 * h); // u * ∂u/∂x (nonlinear convection) - double term_diff = -nu * (u_im1 - 2 * u_i + u_ip1) / (h * h); // -nu * ∂²u/∂x² - - dresidual(i) = term_time + term_conv + term_diff; - } - }; -} - -// Callback function objects for Burger's equation -struct BurgerResidualCallback -{ - size_t N; - double h, dt, nu; - - template - void - operator()(ctx_t& ctx, const vector_t& x, const vector_t& x_prev, vector_t& residual) const - { - compute_residual_full(ctx, x, x_prev, residual, N, h, dt, nu); - } -}; - -struct BurgerJacobianCallback -{ - size_t N; - double h, dt, nu; - - template - void operator()(ctx_t& ctx, const vector_t& x, vector_t& jacobian_values) const - { - assemble_jacobian_full(ctx, x, jacobian_values, N, h, dt, nu); - } -}; - -// Initialize the solution output file (call once at simulation start) -void initialize_solution_file(const char* filename, size_t N, double h) -{ - FILE* fp = fopen(filename, "w"); - if (fp) - { - fprintf(fp, "# Burger equation solution - block format\n"); - fprintf(fp, "# Each timestep is a separate block, separated by blank lines\n"); - fprintf(fp, "# Format: x_coordinate u(x,t)\n"); - fprintf(fp, "# Grid points: %zu, h=%.6e\n", N, h); - fprintf(fp, - "# Use in gnuplot: plot for [i=0:*] 'solution.dat' index i with lines title sprintf('step %%d', i*10)\n"); - fprintf(fp, "#\n"); - fclose(fp); - } - else - { - printf("Error: Could not create %s for writing\n", filename); - } -} - -// Function to append timestep block to solution file (simple and reliable) -template -void dump_solution( - ctx_t& ctx, vector_t& U, size_t timestep, size_t N, double h, double dt, const char* filename = "solution.dat") -{ - ctx.host_launch(U.read()).set_symbol("dump solution")->*[timestep, h, N, dt, filename](auto hU) { - FILE* fp = fopen(filename, "a"); // Simple append - no read/modify/write - if (fp) - { - fprintf(fp, "# Timestep %zu, t=%.6e\n", timestep, timestep * dt); - - for (size_t i = 0; i < N; i++) - { - double x = i * h; - fprintf(fp, "%.10e %.10e\n", x, hU(i)); - } - - fprintf(fp, "\n"); // Blank line to separate datasets - fclose(fp); - - printf("Appended timestep %zu (t=%.4e) to %s\n", timestep, timestep * dt, filename); - } - else - { - printf("Error: Could not open %s for appending\n", filename); - } - }; -} -#endif - -int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) -{ -#if _CCCL_CTK_BELOW(12, 4) - fprintf(stderr, "Waiving test: conditional nodes are only available since CUDA 12.4.\n"); - return 0; -#else - // Usage: ./burger [N] [nsteps] [nu] - // N = Grid points (default: 100000) - // nsteps = Time steps (default: 10000) - // nu = Viscosity (default: 0.05, try 0.001 for shocks) - - stackable_ctx ctx; - - size_t N = 2560; - if (argc > 1) - { - N = atoi(argv[1]); - fprintf(stderr, "N = %zu\n", N); - } - - size_t nsteps = 200; - if (argc > 2) - { - nsteps = atol(argv[2]); - fprintf(stderr, "nsteps = %ld\n", nsteps); - } - - // Set reasonable parameters - implicit method allows larger time steps - double nu = 0.05; // Default viscosity - if (argc > 3) - { - nu = atof(argv[3]); - fprintf(stderr, "nu = %e\n", nu); - } - - ssize_t output_freq = -1; - if (argc > 4) - { - output_freq = atoi(argv[4]); - fprintf(stderr, "output_freq %ld\n", output_freq); - } - - // use_while = 0 => no while; 1 => while in CG, 2 => while in Newton and CG - int use_while = 2; - if (argc > 5) - { - use_while = atoi(argv[5]); - fprintf(stderr, "use_while = %d\n", use_while); - } - - double h = 1.0 / (N - 1); - - double dt_diffusion = 0.5 * h * h / nu; // Diffusion-limited time step - double dt_fixed = 0.001; // Fixed reasonable time step - double dt = std::max(dt_diffusion, dt_fixed); // Use larger of the two - - // For very fine grids, cap the time step to prevent tiny steps - if (N > 10000) - { - dt = std::min(dt, 0.01); // Cap at 0.01 for large grids - } - - double total_time = nsteps * dt; - - fprintf(stderr, "=== Simulation Parameters ===\n"); - fprintf(stderr, "Grid: N=%zu, h=%e\n", N, h); - fprintf(stderr, "Time: dt=%e, nsteps=%zu, total_time=%e\n", dt, nsteps, total_time); - fprintf(stderr, "Physics: nu=%e (viscosity)\n", nu); - fprintf(stderr, "Diffusion number: nu*dt/h^2 = %e\n", nu * dt / (h * h)); - fprintf(stderr, "=============================\n"); - - // Full N×N system: boundary rows have 1 entry each, interior rows have 3 entries each - // Total: 2*1 + (N-2)*3 = 3*N - 4 non-zeros - size_t nz = 3 * N - 4; - - size_t* row_offsets; - size_t* col_indices; - cuda_safe_call(cudaHostAlloc(&row_offsets, (N + 1) * sizeof(size_t), cudaHostAllocMapped)); - cuda_safe_call(cudaHostAlloc(&col_indices, nz * sizeof(size_t), cudaHostAllocMapped)); - - build_full_csr_structure(row_offsets, col_indices, N); - - auto csr_row_offsets = ctx.logical_data(make_slice(row_offsets, N + 1)).set_symbol("csr_row"); - auto csr_col_ind = ctx.logical_data(make_slice(col_indices, nz)).set_symbol("csr_col"); - auto csr_values = ctx.logical_data(shape_of>(nz)).set_symbol("csr_val"); - - auto U = ctx.logical_data(shape_of>(N)).set_symbol("U"); - - // This will prevent erroneous modifications and may allow access from concurrent graphs - csr_row_offsets.set_read_only(); - csr_col_ind.set_read_only(); - - // Initial condition - ctx.parallel_for(U.shape(), U.write()).set_symbol("init conditions")->*[h, N] __device__(size_t i, auto dU) { - double x = i * h; - if (i == 0 || i == N - 1) - { - dU(i) = 0.0; // Homogeneous Dirichlet boundary conditions - } - else - { - dU(i) = sin(M_PI * x); - } - }; - - // Initialize solution output file - initialize_solution_file("solution.dat", N, h); - - auto start = std::chrono::high_resolution_clock::now(); - cuda_safe_call(cudaStreamSynchronize(ctx.fence())); - - // Parameters are now set above with auto-scaling - size_t substeps = (output_freq > 0) ? output_freq : nsteps; - size_t outer_iterations = nsteps / substeps; - - if (use_while == 2) - { - for (size_t outer = 0; outer < outer_iterations; outer++) - { - auto g = ctx.graph_scope(); - - // Repeat substeps inner iterations using STF repeat block - { - auto repeat_guard = ctx.repeat_graph_scope(substeps); - - // Create callback function objects for Burger's equation - BurgerResidualCallback residual_callback{N, h, dt, nu}; - BurgerJacobianCallback jacobian_callback{N, h, dt, nu}; - - // Solve the nonlinear system using generic Newton solver - newton_solver(ctx, U, csr_values, csr_row_offsets, csr_col_ind, residual_callback, jacobian_callback); - } // repeat_guard automatically manages the loop condition - - // Dump solution after each substep block - size_t current_timestep = (outer + 1) * substeps; - dump_solution(ctx, U, current_timestep, N, h, dt); - } - } - else - { - for (size_t outer = 0; outer < outer_iterations; outer++) - { - // Repeat substeps inner iterations using STF repeat block - for (size_t substep = 0; substep < substeps; substep++) - { - // Create callback function objects for Burger's equation - BurgerResidualCallback residual_callback{N, h, dt, nu}; - BurgerJacobianCallback jacobian_callback{N, h, dt, nu}; - - // Solve the nonlinear system using generic Newton solver - newton_solver_no_while( - ctx, U, csr_values, csr_row_offsets, csr_col_ind, residual_callback, jacobian_callback, use_while == 1); - } // repeat_guard automatically manages the loop condition - - // Dump solution after each substep block - size_t current_timestep = (outer + 1) * substeps; - dump_solution(ctx, U, current_timestep, N, h, dt); - } - } - - cuda_safe_call(cudaStreamSynchronize(ctx.fence())); - auto end = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(end - start).count(); - std::cout << "Duration: " << duration << " milliseconds" << '\n'; - - ctx.finalize(); -#endif -} diff --git a/cccl_upstream/cudax/examples/stf/linear_algebra/burger_sensitivity.cu b/cccl_upstream/cudax/examples/stf/linear_algebra/burger_sensitivity.cu deleted file mode 100644 index 5a5c96bf..00000000 --- a/cccl_upstream/cudax/examples/stf/linear_algebra/burger_sensitivity.cu +++ /dev/null @@ -1,479 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Sensitivity analysis for Burger equation - analyze shock formation vs viscosity - */ - -#include - -#include -#include -#include -#include -#include - -#include "cg_solver.cuh" -#include "dot.cuh" -#include "newton_solver.cuh" - -using namespace cuda::experimental::stf; - -#if !_CCCL_CTK_BELOW(12, 4) - -void build_tridiagonal_csr_structure(size_t* row_offsets, size_t* col_indices, size_t N) -{ - size_t nnz = 0; - row_offsets[0] = 0; - - for (size_t row = 0; row < N; row++) - { - if (row == 0 || row == N - 1) - { - // Boundary rows: only diagonal entry (identity for BC: u[i] = prescribed_value) - col_indices[nnz++] = row; - } - else - { - // Interior rows: tridiagonal structure (left, center, right) - col_indices[nnz++] = row - 1; // left neighbor - col_indices[nnz++] = row; // center (diagonal) - col_indices[nnz++] = row + 1; // right neighbor - } - row_offsets[row + 1] = nnz; - } -} - -template -void assemble_jacobian_full( - ctx_t& ctx, vector_t U, vector_t values, size_t N, double h, double dt, double nu) -{ - ctx.parallel_for(box(N), U.read(), values.write()).set_symbol("assemble_jacobian_full") - ->*[N, h, dt, nu] __device__(size_t row, auto dU, auto dvalues) { - if (row == 0) - { - // Left boundary: u[0] = 0 (homogeneous Dirichlet) - size_t val_idx = 0; - dvalues[val_idx] = 1.0; - } - else if (row == N - 1) - { - // Right boundary: u[N-1] = 0 (homogeneous Dirichlet) - size_t val_idx = 1 + 3 * (N - 2); - dvalues[val_idx] = 1.0; - } - else - { - // Interior point: Burger's equation discretization - double u_i = dU[row]; - double u_ip1 = dU[row + 1]; - double u_im1 = dU[row - 1]; - - // Jacobian entries: ∂F_i/∂u_{i-1}, ∂F_i/∂u_i, ∂F_i/∂u_{i+1} - double left = -u_i / (2 * h) - nu / (h * h); - double center = 1.0 / dt + (u_ip1 - u_im1) / (2 * h) + 2.0 * nu / (h * h); - double right = u_i / (2 * h) - nu / (h * h); - - size_t val_idx = 1 + 3 * (row - 1); - dvalues[val_idx] = left; - dvalues[val_idx + 1] = center; - dvalues[val_idx + 2] = right; - } - }; -} - -template -void compute_residual_full( - ctx_t& ctx, vector_t U, vector_t U_prev, vector_t residual, size_t N, double h, double dt, double nu) -{ - ctx.parallel_for(box(N), residual.write(), U.read(), U_prev.read()).set_symbol("compute_residual_full") - ->*[N, h, dt, nu] __device__(size_t i, auto dresidual, auto dU, auto dU_prev) { - if (i == 0) - { - dresidual(i) = dU(i) - 0.0; - } - else if (i == N - 1) - { - dresidual(i) = dU(i) - 0.0; - } - else - { - // Interior point: Burger's equation F_i = ∂u/∂t + u*∂u/∂x - nu*∂²u/∂x² - double u_i = dU(i); - double u_ip1 = dU(i + 1); - double u_im1 = dU(i - 1); - - double term_time = (u_i - dU_prev(i)) / dt; - double term_conv = u_i * (u_ip1 - u_im1) / (2 * h); - double term_diff = -nu * (u_im1 - 2 * u_i + u_ip1) / (h * h); - - dresidual(i) = term_time + term_conv + term_diff; - } - }; -} - -// Shock detection: compute maximum gradient magnitude -template -void detect_shock( - ctx_t& ctx, vector_t& U, stackable_logical_data>& max_gradient, size_t N, double h) -{ - ctx.parallel_for(box(N - 1), U.read(), max_gradient.reduce(reducer::maxval{})).set_symbol("detect_shock") - ->*[h] __device__(size_t i, auto dU, double& dmax_grad) { - double gradient = fabs(dU(i + 1) - dU(i)) / h; - dmax_grad = fmax(dmax_grad, gradient); - }; -} - -// Callback function objects for Burger's equation -struct BurgerResidualCallback -{ - size_t N; - double h, dt, nu; - - template - void - operator()(ctx_t& ctx, const vector_t& x, const vector_t& x_prev, vector_t& residual) const - { - compute_residual_full(ctx, x, x_prev, residual, N, h, dt, nu); - } -}; - -struct BurgerJacobianCallback -{ - size_t N; - double h, dt, nu; - - template - void operator()(ctx_t& ctx, const vector_t& x, vector_t& jacobian_values) const - { - assemble_jacobian_full(ctx, x, jacobian_values, N, h, dt, nu); - } -}; - -// Generate nu values around target with given distribution -std::vector generate_nu_samples(double nu_target, double nu_std, size_t num_samples) -{ - std::vector nu_values; - std::random_device rd; - std::mt19937 gen(rd()); - std::normal_distribution dist(nu_target, nu_std); - - // Generate samples and ensure they are positive - for (size_t i = 0; i < num_samples; ++i) - { - double nu_sample = dist(gen); - // Ensure nu > 0 for physical validity - if (nu_sample > 1e-6) - { - nu_values.push_back(nu_sample); - } - else - { - // Retry if we get non-physical values - i--; - } - } - - // Sort for better output organization - std::sort(nu_values.begin(), nu_values.end()); - return nu_values; -} - -// Initialize sensitivity analysis output file -void initialize_sensitivity_file(const char* filename, double nu_target, double nu_std, size_t num_samples) -{ - FILE* fp = fopen(filename, "w"); - if (fp) - { - fprintf(fp, "# Burger equation sensitivity analysis\n"); - fprintf(fp, "# Target nu: %.6e, std: %.6e, samples: %zu\n", nu_target, nu_std, num_samples); - fprintf(fp, "# Format: nu_value shock_time max_gradient final_time\n"); - fprintf(fp, "# shock_time: time when max gradient exceeds threshold (or -1 if no shock)\n"); - fprintf(fp, "# max_gradient: maximum gradient achieved\n"); - fprintf(fp, "# final_time: total simulation time reached\n"); - fprintf(fp, "#\n"); - fclose(fp); - } -} - -// Initialize shock solutions output file -void initialize_shock_file(const char* filename, double shock_threshold) -{ - FILE* fp = fopen(filename, "w"); - if (fp) - { - fprintf(fp, "# Burger equation shock solutions\n"); - fprintf(fp, "# Solutions dumped when gradient exceeds threshold: %.1f\n", shock_threshold); - fprintf(fp, "# Each shock is a separate data block, separated by blank lines\n"); - fprintf(fp, "# Block header: Sample ID, nu value, shock time, max gradient\n"); - fprintf(fp, "# Block format: x_coordinate u(x,t_shock)\n"); - fprintf(fp, "#\n"); - fprintf(fp, - "# Use in gnuplot: plot for [i=0:*] 'shock_solutions.dat' index i with lines title sprintf('Sample %%d', " - "i+1)\n"); - fprintf(fp, "#\n"); - fclose(fp); - printf("Initialized shock solutions file: %s\n", filename); - } -} - -// Dump solution when shock is detected -template -void dump_shock_solution( - ctx_t& ctx, - vector_t& U, - double nu, - double shock_time, - double max_gradient, - size_t sample_id, - size_t N, - double h, - const char* filename = "shock_solutions.dat") -{ - ctx.host_launch(U.read()).set_symbol("dump shock solution") - ->* - [nu, shock_time, max_gradient, sample_id, h, N, filename](auto hU) { - FILE* fp = fopen(filename, "a"); // Append to file - if (fp) - { - fprintf( - fp, "# Sample %zu: nu=%.6e, shock_time=%.6e, max_gradient=%.2f\n", sample_id, nu, shock_time, max_gradient); - fprintf(fp, "# Format: x_coordinate u(x,t_shock)\n"); - - for (size_t i = 0; i < N; i++) - { - double x = i * h; - fprintf(fp, "%.10e %.10e\n", x, hU(i)); - } - - fprintf(fp, "\n"); // Blank line to separate datasets - fclose(fp); - - printf(" -> Solution saved to %s", filename); - } - else - { - printf(" -> Error: Could not save solution to %s", filename); - } - }; -} - -template -void run_single_nu_simulation( - ctx_t& ctx, - double nu, - vector_t& U, - vector_t& csr_values, - const vector_t& csr_row_offsets, - const vector_t& csr_col_ind, - size_t N, - double h, - double dt, - double max_time, - double shock_threshold, - size_t sample_id, - double& shock_time, - double& max_gradient, - double& final_time) -{ - // Reset solution to initial condition - ctx.parallel_for(U.shape(), U.write()).set_symbol("reset_initial_condition")->*[h, N] __device__(size_t i, auto dU) { - double x = i * h; - dU(i) = (i == 0 || i == N - 1) ? 0.0 : sin(M_PI * x); - }; - - auto current_time = ctx.logical_data(shape_of>()).set_symbol("current_time"); - auto max_grad_global = ctx.logical_data(shape_of>()).set_symbol("max_grad_global"); - auto shock_detected = ctx.logical_data(shape_of>()).set_symbol("shock_detected"); - - // Initialize tracking variables - ctx.parallel_for(box(1), current_time.write(), max_grad_global.write(), shock_detected.write()) - .set_symbol("init_tracking") - ->*[] __device__(size_t i, auto dtime, auto dmax_grad, auto dshock) { - *dtime = 0.0; - *dmax_grad = 0.0; - *dshock = 0; // 0 = no shock, 1 = shock detected - }; - - // Time evolution loop with shock detection - { - auto while_guard = ctx.while_graph_scope(); - - // Create callback function objects - BurgerResidualCallback residual_callback{N, h, dt, nu}; - BurgerJacobianCallback jacobian_callback{N, h, dt, nu}; - - // Solve the nonlinear system - newton_solver(ctx, U, csr_values, csr_row_offsets, csr_col_ind, residual_callback, jacobian_callback); - - // Update time - ctx.parallel_for(box(1), current_time.rw()).set_symbol("update_time")->*[dt] __device__(size_t i, auto dtime) { - *dtime += dt; - }; - - // Detect shock by computing maximum gradient - auto current_grad = ctx.logical_data(shape_of>()).set_symbol("current_grad"); - detect_shock(ctx, U, current_grad, N, h); - - // Update global maximum gradient and check for shock - ctx.parallel_for(box(1), max_grad_global.rw(), current_grad.read(), shock_detected.rw()) - .set_symbol("update_shock_detection") - ->*[shock_threshold] __device__(size_t i, auto dmax_grad, auto dcurrent_grad, auto dshock) { - double grad = *dcurrent_grad; - if (grad > *dmax_grad) - { - *dmax_grad = grad; - } - if (grad > shock_threshold && *dshock == 0) - { - *dshock = 1; // First time shock threshold is exceeded - } - }; - - // Continue while time < max_time and no shock detected - while_guard.update_cond(current_time.read(), shock_detected.read())->*[max_time] __device__(auto dtime, auto dshock) { - return (*dtime < max_time) && (*dshock == 0); - }; - } - - // Extract results to host variables - ctx.host_launch(current_time.read(), max_grad_global.read(), shock_detected.read()).set_symbol("extract_results") - ->*[&shock_time, &max_gradient, &final_time](auto htime, auto hmax_grad, auto hshock) { - final_time = *htime; - max_gradient = *hmax_grad; - shock_time = (*hshock == 1) ? *htime : -1.0; // -1 indicates no shock - - if (shock_time > 0) - { - printf("shock at t=%.4f, max_grad=%.1f\n", shock_time, max_gradient); - } - else - { - printf("no shock, max_grad=%.1f\n", max_gradient); - } - }; - - // Dump solution if shock was detected - if (shock_time > 0) - { - dump_shock_solution(ctx, U, nu, shock_time, max_gradient, sample_id, N, h); - } -} - -#endif - -int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) -{ -#if _CCCL_CTK_BELOW(12, 4) - fprintf(stderr, "Waiving test: conditional nodes are only available since CUDA 12.4.\n"); - return 0; -#else - // Usage: ./burger_sensitivity [N] [nu_target] [nu_std] [num_samples] [shock_threshold] - - size_t N = 1000; // Smaller grid for sensitivity analysis - double nu_target = 0.02; // Target viscosity - double nu_std = 0.01; // Standard deviation for nu distribution - size_t num_samples = 20; // Number of nu samples to test - - double shock_threshold = 15.0; // Gradient threshold to detect shock (du/dx magnitude) - - if (argc > 1) - { - N = atoi(argv[1]); - } - if (argc > 2) - { - nu_target = atof(argv[2]); - } - if (argc > 3) - { - nu_std = atof(argv[3]); - } - if (argc > 4) - { - num_samples = atoi(argv[4]); - } - if (argc > 5) - { - shock_threshold = atof(argv[5]); - } - - double h = 1.0 / (N - 1); - double dt = 0.001; // Fixed time step - double max_time = 2.0; // Maximum simulation time per sample - - fprintf(stderr, "=== Sensitivity Analysis Parameters ===\n"); - fprintf(stderr, "Grid: N=%zu, h=%e\n", N, h); - fprintf(stderr, "Viscosity: target=%e, std=%e, samples=%zu\n", nu_target, nu_std, num_samples); - fprintf(stderr, "Time: dt=%e, max_time=%e\n", dt, max_time); - fprintf(stderr, "Shock threshold: %.1f (gradient magnitude)\n", shock_threshold); - fprintf(stderr, "======================================\n"); - - stackable_ctx ctx; - - // Generate nu samples - auto nu_values = generate_nu_samples(nu_target, nu_std, num_samples); - - // Set up CSR structure - size_t nz = 3 * N - 4; - size_t* row_offsets; - size_t* col_indices; - cuda_safe_call(cudaHostAlloc(&row_offsets, (N + 1) * sizeof(size_t), cudaHostAllocMapped)); - cuda_safe_call(cudaHostAlloc(&col_indices, nz * sizeof(size_t), cudaHostAllocMapped)); - build_tridiagonal_csr_structure(row_offsets, col_indices, N); - - auto csr_row_offsets = ctx.logical_data(make_slice(row_offsets, N + 1)).set_symbol("csr_row"); - auto csr_col_ind = ctx.logical_data(make_slice(col_indices, nz)).set_symbol("csr_col"); - csr_row_offsets.set_read_only(); - csr_col_ind.set_read_only(); - - // Initialize output files - initialize_sensitivity_file("sensitivity_results.dat", nu_target, nu_std, num_samples); - initialize_shock_file("shock_solutions.dat", shock_threshold); - - // Run sensitivity analysis - printf("Running sensitivity analysis with %zu samples...\n", num_samples); - - { - auto g = ctx.graph_scope(); - for (size_t i = 0; i < nu_values.size(); ++i) - { - double nu = nu_values[i]; - double shock_time, max_gradient, final_time; - - printf("Sample %zu/%zu: nu=%.6e... ", i + 1, nu_values.size(), nu); - fflush(stdout); - - auto csr_values = ctx.logical_data(shape_of>(nz)).set_symbol("csr_val"); - auto U = ctx.logical_data(shape_of>(N)).set_symbol("U"); - - run_single_nu_simulation( - ctx, - nu, - U, - csr_values, - csr_row_offsets, - csr_col_ind, - N, - h, - dt, - max_time, - shock_threshold, - i + 1, - shock_time, - max_gradient, - final_time); - } - } - - ctx.finalize(); -#endif -} diff --git a/cccl_upstream/cudax/examples/stf/linear_algebra/cg_csr.cu b/cccl_upstream/cudax/examples/stf/linear_algebra/cg_csr.cu deleted file mode 100644 index e88e6b57..00000000 --- a/cccl_upstream/cudax/examples/stf/linear_algebra/cg_csr.cu +++ /dev/null @@ -1,232 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Sparse conjugate gradient algorithm - */ - -#include - -using namespace cuda::experimental::stf; - -using vector_t = logical_data>; -using scalar_t = logical_data>; -using context_t = context; - -struct csr_matrix -{ - csr_matrix( - context_t& ctx, size_t num_rows, size_t num_nonzeros, double* values, size_t* row_offsets, size_t* column_indices) - { - val_handle = ctx.logical_data(make_slice(values, num_nonzeros)); - col_handle = ctx.logical_data(make_slice(column_indices, num_nonzeros)); - row_handle = ctx.logical_data(make_slice(row_offsets, num_rows + 1)); - } - - /* Description of the CSR */ - mutable logical_data> val_handle; - mutable logical_data> row_handle; - mutable logical_data> col_handle; -}; - -// Note that a and b might be the same logical data -void DOT(context_t& ctx, vector_t& a, vector_t& b, scalar_t& res) -{ - ctx.parallel_for(a.shape(), a.read(), b.read(), res.reduce(reducer::sum{})) - ->*[] __device__(size_t i, auto da, auto db, double& dres) { - dres += da(i) * db(i); - }; -}; - -void SPMV(context_t& ctx, csr_matrix& a, vector_t& x, vector_t& y) -{ - ctx.parallel_for(y.shape(), a.val_handle.read(), a.col_handle.read(), a.row_handle.read(), x.read(), y.write()) - ->*[] _CCCL_DEVICE(size_t row, auto da_val, auto da_col, auto da_row, auto dx, auto dy) { - int row_start = da_row(row); - int row_end = da_row(row + 1); - - double sum = 0.0; - for (int elt = row_start; elt < row_end; elt++) - { - sum += da_val(elt) * dx(da_col(elt)); - } - - dy(row) = sum; - }; -} - -/* genTridiag: generate a random tridiagonal symmetric matrix - from : - https://github.com/NVIDIA/cuda-samples/blob/master/Samples/4_CUDA_Libraries/conjugateGradientCudaGraphs/conjugateGradientCudaGraphs.cu - */ -void genTridiag(size_t* I, size_t* J, double* val, size_t N, size_t nz) -{ - const double d = 2.0; - - I[0] = 0, J[0] = 0, J[1] = 1; - val[0] = drand48() + d; - val[1] = drand48(); - int start; - - for (size_t i = 1; i < N; i++) - { - if (i > 1) - { - I[i] = I[i - 1] + 3; - } - else - { - I[1] = 2; - } - - start = (i - 1) * 3 + 2; - J[start] = i - 1; - J[start + 1] = i; - - if (i < N - 1) - { - J[start + 2] = i + 1; - } - - val[start] = val[start - 1]; - val[start + 1] = drand48() + d; - - if (i < N - 1) - { - val[start + 2] = drand48(); - } - } - - I[N] = nz; -} - -void cg_solver(context_t& ctx, csr_matrix& A, vector_t& X, vector_t& B) -{ - // Initial guess X = 1 - ctx.parallel_for(X.shape(), X.write())->*[] _CCCL_DEVICE(size_t i, auto dX) { - dX(i) = 1.0; - }; - - // Residual R initialized to B - auto R = ctx.logical_data(B.shape()); - ctx.parallel_for(R.shape(), R.write(), B.read())->*[] _CCCL_DEVICE(size_t i, auto dR, auto dB) { - dR(i) = dB(i); - }; - - // R = R - A*X - auto Ax = ctx.logical_data(X.shape()); - SPMV(ctx, A, X, Ax); - ctx.parallel_for(R.shape(), R.rw(), Ax.read())->*[] _CCCL_DEVICE(size_t i, auto dR, auto dAx) { - dR(i) -= dAx(i); - }; - - // P = R; - auto P = ctx.logical_data(R.shape()); - ctx.parallel_for(P.shape(), P.write(), R.read())->*[] _CCCL_DEVICE(size_t i, auto dP, auto dR) { - dP(i) = dR(i); - }; - - // RSOLD = R'*R - auto rsold = ctx.logical_data(shape_of>()); - DOT(ctx, R, R, rsold); - - const int MAXITER = X.shape().size(); - for (int k = 0; k < MAXITER; k++) - { - // Ap = A*P - auto Ap = ctx.logical_data(P.shape()); - SPMV(ctx, A, P, Ap); - - // We don't compute alpha explicitly - // alpha = rsold / (p' * Ap); - auto pAp = ctx.logical_data(shape_of>()); - DOT(ctx, P, Ap, pAp); - - // x = x + alpha * p; - ctx.parallel_for(X.shape(), X.rw(), rsold.read(), pAp.read(), P.read()) - ->*[] _CCCL_DEVICE(size_t i, auto dX, auto drsold, auto dpAp, auto dP) { - double alpha = (*drsold / *dpAp); - dX(i) += alpha * dP(i); - }; - - // r = r - alpha * Ap; - ctx.parallel_for(R.shape(), R.rw(), rsold.read(), pAp.read(), Ap.read()) - ->*[] _CCCL_DEVICE(size_t i, auto dR, auto drsold, auto dpAp, auto dAp) { - double alpha = (*drsold / *dpAp); - dR(i) -= alpha * dAp(i); - }; - - // rsnew = r' * r; - auto rsnew = ctx.logical_data(shape_of>()); - DOT(ctx, R, R, rsnew); - - // Read the residual on the CPU, and halt the iterative process if we have converged - // (note that this will block the submission of tasks) - double err = ctx.wait(rsnew); - fprintf(stderr, "iter %d : residual %e\n", k, err); - if (err < 1e-10) - { - // We have converged - fprintf(stderr, "Successfully converged (err = %le)\n", err); - break; - } - - // p = r + (rsnew / rsold) * p; - ctx.parallel_for(P.shape(), P.rw(), R.read(), rsnew.read(), rsold.read()) - ->*[] _CCCL_DEVICE(size_t i, auto dP, auto dR, auto drsnew, auto drsold) { - dP(i) = dR(i) + (*drsnew / *drsold) * dP(i); - }; - - // update old residual - ctx.parallel_for(box(1), rsold.write(), rsnew.read())->*[] _CCCL_DEVICE(size_t i, auto drsold, auto drsnew) { - *drsold = *drsnew; - }; - } -} - -int main(int argc, char** argv) -{ - size_t N = 10485760; - - context_t ctx; - - if (argc > 1) - { - N = atoi(argv[1]); - fprintf(stderr, "N = %zu\n", N); - } - - size_t nz = (N - 2) * 3 + 4; - - size_t* row_offsets; - size_t* column_indices; - double* values; - cuda_safe_call(cudaHostAlloc(&row_offsets, (N + 1) * sizeof(size_t), cudaHostAllocMapped)); - cuda_safe_call(cudaHostAlloc(&column_indices, nz * sizeof(size_t), cudaHostAllocMapped)); - cuda_safe_call(cudaHostAlloc(&values, nz * sizeof(double), cudaHostAllocMapped)); - - // Generate a random matrix that is supposed to be invertible - genTridiag(row_offsets, column_indices, values, N, nz); - - csr_matrix A(ctx, N, nz, values, row_offsets, column_indices); - - auto X = ctx.logical_data(shape_of>(N)); - auto B = ctx.logical_data(shape_of>(N)); - - // RHS - ctx.parallel_for(B.shape(), B.write())->*[] __device__(size_t i, auto dB) { - dB(i) = 1.0; - }; - - cg_solver(ctx, A, X, B); - - ctx.finalize(); -} diff --git a/cccl_upstream/cudax/examples/stf/linear_algebra/cg_csr_stackable.cu b/cccl_upstream/cudax/examples/stf/linear_algebra/cg_csr_stackable.cu deleted file mode 100644 index 4720a8e9..00000000 --- a/cccl_upstream/cudax/examples/stf/linear_algebra/cg_csr_stackable.cu +++ /dev/null @@ -1,240 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Sparse conjugate gradient algorithm - */ - -#include - -using namespace cuda::experimental::stf; - -#if !_CCCL_CTK_BELOW(12, 4) -using vector_t = stackable_logical_data>; -using scalar_t = stackable_logical_data>; -using context_t = stackable_ctx; - -struct csr_matrix -{ - csr_matrix( - context_t& ctx, size_t num_rows, size_t num_nonzeros, double* values, size_t* row_offsets, size_t* column_indices) - { - val_handle = ctx.logical_data(make_slice(values, num_nonzeros)); - col_handle = ctx.logical_data(make_slice(column_indices, num_nonzeros)); - row_handle = ctx.logical_data(make_slice(row_offsets, num_rows + 1)); - - val_handle.set_symbol("csr_val"); - col_handle.set_symbol("csr_col"); - row_handle.set_symbol("csr_row"); - } - - /* Description of the CSR */ - mutable stackable_logical_data> val_handle; - mutable stackable_logical_data> row_handle; - mutable stackable_logical_data> col_handle; -}; - -// Note that a and b might be the same logical data -void DOT(context_t& ctx, vector_t& a, vector_t& b, scalar_t& res) -{ - ctx.parallel_for(a.shape(), a.read(), b.read(), res.reduce(reducer::sum{})).set_symbol("DOT")->* - [] __device__(size_t i, auto da, auto db, double& dres) { - dres += da(i) * db(i); - }; -}; - -void SPMV(context_t& ctx, csr_matrix& a, vector_t& x, vector_t& y) -{ - ctx.parallel_for(y.shape(), a.val_handle.read(), a.col_handle.read(), a.row_handle.read(), x.read(), y.write()) - .set_symbol("SPMV") - ->*[] _CCCL_DEVICE(size_t row, auto da_val, auto da_col, auto da_row, auto dx, auto dy) { - int row_start = da_row(row); - int row_end = da_row(row + 1); - - double sum = 0.0; - for (int elt = row_start; elt < row_end; elt++) - { - sum += da_val(elt) * dx(da_col(elt)); - } - - dy(row) = sum; - }; -} - -/* genTridiag: generate a random tridiagonal symmetric matrix - from : - https://github.com/NVIDIA/cuda-samples/blob/master/Samples/4_CUDA_Libraries/conjugateGradientCudaGraphs/conjugateGradientCudaGraphs.cu - */ -void genTridiag(size_t* I, size_t* J, double* val, size_t N, size_t nz) -{ - const double d = 2.0; - - I[0] = 0, J[0] = 0, J[1] = 1; - val[0] = drand48() + d; - val[1] = drand48(); - int start; - - for (size_t i = 1; i < N; i++) - { - if (i > 1) - { - I[i] = I[i - 1] + 3; - } - else - { - I[1] = 2; - } - - start = (i - 1) * 3 + 2; - J[start] = i - 1; - J[start + 1] = i; - - if (i < N - 1) - { - J[start + 2] = i + 1; - } - - val[start] = val[start - 1]; - val[start + 1] = drand48() + d; - - if (i < N - 1) - { - val[start + 2] = drand48(); - } - } - - I[N] = nz; -} - -void cg_solver(context_t& ctx, csr_matrix& A, vector_t& X, vector_t& B) -{ - // Initial guess X = 1 - ctx.parallel_for(X.shape(), X.write()).set_symbol("init_guess")->*[] _CCCL_DEVICE(size_t i, auto dX) { - dX(i) = 1.0; - }; - - // Residual R initialized to B - auto R = ctx.logical_data(B.shape()); - ctx.parallel_for(R.shape(), R.write(), B.read()).set_symbol("R=B")->*[] _CCCL_DEVICE(size_t i, auto dR, auto dB) { - dR(i) = dB(i); - }; - - // R = R - A*X - auto Ax = ctx.logical_data(X.shape()).set_symbol("Ax"); - SPMV(ctx, A, X, Ax); - ctx.parallel_for(R.shape(), R.rw(), Ax.read()).set_symbol("R -= Ax")->*[] _CCCL_DEVICE(size_t i, auto dR, auto dAx) { - dR(i) -= dAx(i); - }; - - // P = R; - auto P = ctx.logical_data(R.shape()).set_symbol("P"); - ctx.parallel_for(P.shape(), P.write(), R.read()).set_symbol("P=R")->*[] _CCCL_DEVICE(size_t i, auto dP, auto dR) { - dP(i) = dR(i); - }; - - // RSOLD = R'*R - auto rsold = ctx.logical_data(shape_of>()).set_symbol("rsold"); - DOT(ctx, R, R, rsold); - - { - auto while_guard = ctx.while_graph_scope(); - - // Ap = A*P - auto Ap = ctx.logical_data(P.shape()).set_symbol("Ap"); - SPMV(ctx, A, P, Ap); - - // We don't compute alpha explicitly - // alpha = rsold / (p' * Ap); - auto pAp = ctx.logical_data(shape_of>()).set_symbol("pAp"); - DOT(ctx, P, Ap, pAp); - - // x = x + alpha * p; - ctx.parallel_for(X.shape(), X.rw(), rsold.read(), pAp.read(), P.read()).set_symbol("X+=alpha*P") - ->*[] _CCCL_DEVICE(size_t i, auto dX, auto drsold, auto dpAp, auto dP) { - double alpha = (*drsold / *dpAp); - dX(i) += alpha * dP(i); - }; - - // r = r - alpha * Ap; - ctx.parallel_for(R.shape(), R.rw(), rsold.read(), pAp.read(), Ap.read()).set_symbol("R-=alpha*Ap") - ->*[] _CCCL_DEVICE(size_t i, auto dR, auto drsold, auto dpAp, auto dAp) { - double alpha = (*drsold / *dpAp); - dR(i) -= alpha * dAp(i); - }; - - // rsnew = r' * r; - auto rsnew = ctx.logical_data(shape_of>()).set_symbol("rsnew"); - DOT(ctx, R, R, rsnew); - - while_guard.update_cond(rsnew.read())->*[] __device__(auto drsnew) { - printf("RES %e\n", *drsnew); - bool converged = (*drsnew < 1e-13); - return !converged; - }; - - // p = r + (rsnew / rsold) * p; - ctx.parallel_for(P.shape(), P.rw(), R.read(), rsnew.read(), rsold.read()).set_symbol("P=r+(rsnew/rsold)*P") - ->*[] _CCCL_DEVICE(size_t i, auto dP, auto dR, auto drsnew, auto drsold) { - dP(i) = dR(i) + (*drsnew / *drsold) * dP(i); - }; - - // update old residual - ctx.parallel_for(box(1), rsold.write(), rsnew.read()).set_symbol("update_rsold") - ->*[] _CCCL_DEVICE(size_t i, auto drsold, auto drsnew) { - *drsold = *drsnew; - }; - } -} -#endif - -int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) -{ -#if _CCCL_CTK_BELOW(12, 4) - fprintf(stderr, "Waiving test: conditional nodes are only available since CUDA 12.4.\n"); - return 0; -#else - size_t N = 10485760; - - context_t ctx; - - if (argc > 1) - { - N = atoi(argv[1]); - fprintf(stderr, "N = %zu\n", N); - } - - size_t nz = (N - 2) * 3 + 4; - - size_t* row_offsets; - size_t* column_indices; - double* values; - cuda_safe_call(cudaHostAlloc(&row_offsets, (N + 1) * sizeof(size_t), cudaHostAllocMapped)); - cuda_safe_call(cudaHostAlloc(&column_indices, nz * sizeof(size_t), cudaHostAllocMapped)); - cuda_safe_call(cudaHostAlloc(&values, nz * sizeof(double), cudaHostAllocMapped)); - - // Generate a random matrix that is supposed to be invertible - genTridiag(row_offsets, column_indices, values, N, nz); - - csr_matrix A(ctx, N, nz, values, row_offsets, column_indices); - - auto X = ctx.logical_data(shape_of>(N)).set_symbol("X"); - auto B = ctx.logical_data(shape_of>(N)).set_symbol("B"); - - // RHS - ctx.parallel_for(B.shape(), B.write()).set_symbol("B assembly")->*[] __device__(size_t i, auto dB) { - dB(i) = 1.0; - }; - - cg_solver(ctx, A, X, B); - - ctx.finalize(); -#endif -} diff --git a/cccl_upstream/cudax/examples/stf/linear_algebra/cg_dense_2D.cu b/cccl_upstream/cudax/examples/stf/linear_algebra/cg_dense_2D.cu deleted file mode 100644 index 201ce11b..00000000 --- a/cccl_upstream/cudax/examples/stf/linear_algebra/cg_dense_2D.cu +++ /dev/null @@ -1,462 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Conjugate gradient for a tiled dense matrix - */ - -#include - -using namespace cuda::experimental::stf; - -static cublasHandle_t cublas_handle; - -stream_ctx ctx; - -class matrix -{ -public: - matrix(size_t N) - : N(N) - { - h_addr.reset(new double[N * N]); - cuda_safe_call(cudaHostRegister(h_addr.get(), N * N * sizeof(double), cudaHostRegisterPortable)); - handle = ::std::make_shared>>( - ctx.logical_data(make_slice(h_addr.get(), std::tuple{N, N}, N))); - } - - void fill(const std::function& f) - { - ctx.task(exec_place::host(), handle->write())->*[&f](cudaStream_t stream, auto ds) { - cuda_safe_call(cudaStreamSynchronize(stream)); - - for (size_t col = 0; col < ds.extent(1); col++) - { - for (size_t row = 0; row < ds.extent(0); row++) - { - ds(row, col) = f(row, col); - } - } - }; - } - - size_t N; - std::unique_ptr h_addr; - std::shared_ptr>> handle; -}; - -class vector -{ -public: - vector(size_t N, size_t _block_size, bool is_tmp = false) - : N(N) - , block_size(_block_size) - , nblocks((N + block_size - 1) / block_size) - { - handles.resize(nblocks); - - if (is_tmp) - { - // There is no physical backing for this temporary vector - for (size_t b = 0; b < nblocks; b++) - { - size_t bs = std::min(N - block_size * b, block_size); - handles[b] = ::std::make_shared>>(ctx.logical_data(shape_of>(bs))); - } - } - else - { - h_addr.reset(new double[N]); - cuda_safe_call(cudaHostRegister(h_addr.get(), N * sizeof(double), cudaHostRegisterPortable)); - for (size_t b = 0; b < nblocks; b++) - { - size_t bs = std::min(N - block_size * b, block_size); - handles[b] = - ::std::make_shared>>(ctx.logical_data(make_slice(&h_addr[block_size * b], bs))); - } - } - } - - // Copy constructor - vector(const vector& a) - : N(a.N) - , block_size(a.block_size) - , nblocks(a.nblocks) - { - handles.resize(nblocks); - - for (size_t b = 0; b < nblocks; b++) - { - size_t bs = std::min(N - block_size * b, block_size); - handles[b] = ::std::make_shared>>(ctx.logical_data(shape_of>(bs))); - - ctx.task(handles[b]->write(), a.handles[b]->read())->*[bs](cudaStream_t stream, auto dthis, auto da) { - // There are likely much more efficient ways. - cuda_safe_call(cudaMemcpyAsync( - dthis.data_handle(), da.data_handle(), bs * sizeof(double), cudaMemcpyDeviceToDevice, stream)); - }; - } - } - - void fill(const std::function& f) - { - size_t bs = block_size; - for (size_t b = 0; b < nblocks; b++) - { - ctx.task(exec_place::host(), handles[b]->write())->*[&f, b, bs](cudaStream_t stream, auto ds) { - cuda_safe_call(cudaStreamSynchronize(stream)); - - for (size_t local_row = 0; local_row < ds.extent(0); local_row++) - { - ds(local_row) = f(local_row + b * bs); - } - }; - } - } - - size_t N; - size_t block_size; - size_t nblocks; - - mutable std::vector>>> handles; - std::unique_ptr h_addr; -}; - -__global__ void scalar_div(const double* a, const double* b, double* c) -{ - *c = *a / *b; -} - -// A += B -__global__ void scalar_add(double* a, const double* b) -{ - *a = *a + *b; -} - -__global__ void scalar_minus(const double* a, double* res) -{ - *res = -(*a); -} - -class scalar -{ -public: - scalar(bool is_tmp = false) - { - size_t s = sizeof(double); - - if (is_tmp) - { - // There is no physical backing for this temporary vector - handle = ::std::make_shared>>(ctx.logical_data(shape_of>(1))); - } - else - { - h_addr.reset(new double); - cuda_safe_call(cudaHostRegister(h_addr.get(), s, cudaHostRegisterPortable)); - handle = ::std::make_shared>>(ctx.logical_data(make_slice(h_addr.get(), 1))); - } - } - - scalar(scalar&&) = default; - scalar& operator=(scalar&&) = default; - - // Copy constructor - scalar(const scalar& a) - { - handle = ::std::make_shared>>(ctx.logical_data(shape_of>(1))); - - ctx.task(handle->write(), a.handle->read())->*[](cudaStream_t stream, auto dthis, auto da) { - // There are likely much more efficient ways. - cuda_safe_call( - cudaMemcpyAsync(dthis.data_handle(), da.data_handle(), sizeof(double), cudaMemcpyDeviceToDevice, stream)); - }; - } - - scalar operator/(scalar const& rhs) const - { - // Submit a task that computes this/rhs - scalar res(true); - ctx.task(handle->read(), rhs.handle->read(), res.handle->write()) - ->*[](cudaStream_t stream, auto da, auto db, auto dres) { - scalar_div<<<1, 1, 0, stream>>>(da.data_handle(), db.data_handle(), dres.data_handle()); - }; - - return res; - } - - // this += rhs - scalar& operator+=(const scalar& rhs) - { - ctx.task(handle->rw(), rhs.handle->read())->*[](cudaStream_t stream, auto dthis, auto drhs) { - scalar_add<<<1, 1, 0, stream>>>(dthis.data_handle(), drhs.data_handle()); - }; - - return *this; - } - - scalar operator-() const - { - // Submit a task that computes -s - scalar res(true); - ctx.task(handle->read(), res.handle->write())->*[](cudaStream_t stream, auto dthis, auto dres) { - scalar_minus<<<1, 1, 0, stream>>>(dthis.data_handle(), dres.data_handle()); - }; - - return res; - } - - // Get value on the host - double get_value() - { - double val; - ctx.task(exec_place::host(), handle->read())->*[&val](cudaStream_t stream, auto ds) { - cuda_safe_call(cudaStreamSynchronize(stream)); - val = ds(0); - }; - - return val; - } - - mutable std::shared_ptr>> handle; - std::unique_ptr h_addr; -}; - -class scalar DOT(vector& a, class vector& b) -{ - assert(a.nblocks == b.nblocks); - scalar global_res(true); - - // Loop over all blocks, - for (size_t bid = 0; bid < a.nblocks; bid++) - { - scalar res(true); - - // Note that it works even if a.handle == b.handle because they have the same access mode - ctx.task(a.handles[bid]->read(), b.handles[bid]->read(), res.handle->write()) - ->*[](cudaStream_t stream, auto da, auto db, auto dres) { - cuda_safe_call(cublasSetStream(cublas_handle, stream)); - cuda_safe_call(cublasSetPointerMode(cublas_handle, CUBLAS_POINTER_MODE_DEVICE)); - cuda_safe_call( - cublasDdot(cublas_handle, da.extent(0), da.data_handle(), 1, db.data_handle(), 1, dres.data_handle())); - }; - - if (bid == 0) - { - // First access requires an assignment because it was not initialized - global_res = std::move(res); - } - else - { - global_res += res; - } - } - - return global_res; -}; - -// Y = Y + alpha * X -void AXPY(const class scalar& alpha, class vector& x, class vector& y) -{ - assert(x.N == y.N); - assert(x.nblocks == y.nblocks); - - for (size_t b = 0; b < x.nblocks; b++) - { - ctx.task(alpha.handle->read(), x.handles[b]->read(), y.handles[b]->rw()) - ->* - [](cudaStream_t stream, auto dalpha, auto dx, auto dy) { - auto nx = dx.extent(0); - cuda_safe_call(cublasSetStream(cublas_handle, stream)); - cuda_safe_call(cublasSetPointerMode(cublas_handle, CUBLAS_POINTER_MODE_DEVICE)); - cuda_safe_call(cublasDaxpy(cublas_handle, nx, dalpha.data_handle(), dx.data_handle(), 1, dy.data_handle(), 1)); - }; - } -}; - -// Y = alpha*Y + X -void SCALE_AXPY(const scalar& alpha, const class vector& x, class vector& y) -{ - assert(x.N == y.N); - assert(x.nblocks == y.nblocks); - - for (size_t b = 0; b < x.nblocks; b++) - { - ctx.task(alpha.handle->read(), x.handles[b]->read(), y.handles[b]->rw()) - ->*[](cudaStream_t stream, auto dalpha, auto dx, auto dy) { - cuda_safe_call(cublasSetStream(cublas_handle, stream)); - - auto nx = dx.extent(0); - - // Y = alpha Y - cuda_safe_call(cublasSetPointerMode(cublas_handle, CUBLAS_POINTER_MODE_DEVICE)); - cuda_safe_call(cublasDscal(cublas_handle, nx, dalpha.data_handle(), dy.data_handle(), 1)); - - // Y = Y + X - const double one = 1.0; - cuda_safe_call(cublasSetPointerMode(cublas_handle, CUBLAS_POINTER_MODE_HOST)); - cuda_safe_call(cublasDaxpy(cublas_handle, nx, &one, dx.data_handle(), 1, dy.data_handle(), 1)); - }; - } -}; - -// y = alpha Ax + beta y -void GEMV(double alpha, class matrix& a, class vector& x, double beta, class vector& y) -{ - assert(a.N == x.N); - assert(x.N == y.N); - - size_t block_size = x.block_size; - assert(block_size == y.block_size); - - for (size_t row_y = 0; row_y < y.nblocks; row_y++) - { - for (size_t row_x = 0; row_x < x.nblocks; row_x++) - { - double local_beta = (row_x == 0) ? beta : 1.0; - - // If beta is null, then this is a write only mode - auto y_mode = local_beta == 0.0 ? access_mode::write : access_mode::rw; - - ctx.task(a.handle->read(), x.handles[row_x]->read(), task_dep>(*(y.handles[row_y].get()), y_mode)) - ->*[alpha, local_beta, row_x, row_y, block_size](cudaStream_t stream, auto da, auto dx, auto dy) { - auto nx = dx.extent(0); - auto ny = dy.extent(0); - auto ldA = da.stride(1); - const double* Ablock = &da(row_y * block_size, row_x * block_size); - - cuda_safe_call(cublasSetStream(cublas_handle, stream)); - cuda_safe_call(cublasSetPointerMode(cublas_handle, CUBLAS_POINTER_MODE_HOST)); - cuda_safe_call(cublasDgemv( - cublas_handle, - CUBLAS_OP_N, - ny, - nx, - &alpha, - Ablock, - ldA, - dx.data_handle(), - 1, - &local_beta, - dy.data_handle(), - 1)); - }; - } - } -} - -void cg(matrix& A, vector& X, vector& B) -{ - int N = A.N; - - assert(N == X.N); - assert(N == B.N); - - vector R = B; - - // R = R - A*X - GEMV(-1.0, A, X, 1.0, R); - - vector P = R; - - // RSOLD = R'*R - scalar rsold = DOT(R, R); - - int MAXITER = N; - - if (getenv("MAXITER")) - { - MAXITER = atoi(getenv("MAXITER")); - } - - for (int k = 0; k < MAXITER; k++) - { - vector Ap(N, P.block_size, true); - - // Ap = A*P - GEMV(1.0, A, P, 0.0, Ap); - - // alpha = rsold / (p' * Ap); - scalar alpha = rsold / DOT(P, Ap); - - // x = x + alpha * p; - AXPY(alpha, P, X); - - // r = r - alpha * Ap; - AXPY(-alpha, Ap, R); - - // rsnew = r' * r; - scalar rsnew = DOT(R, R); - - // Read the residual on the CPU, and halt the iterative process if we have converged - { - double err; - ctx.task(exec_place::host(), rsnew.handle->read())->*[&err](cudaStream_t stream, auto dres) { - cuda_safe_call(cudaStreamSynchronize(stream)); - err = sqrt(dres(0)); - }; - - if (err < 1e-10) - { - // We have converged - // fprintf(stderr, "Successfully converged (err = %le)\n", err); - break; - } - } - - // p = r + (rsnew / rsold) * p; - SCALE_AXPY(rsnew / rsold, R, P); - - rsold = std::move(rsnew); - } -} - -int main(int argc, char** argv) -{ - size_t N = 1024; - - if (argc > 1) - { - N = atoi(argv[1]); - fprintf(stderr, "N = %zu\n", N); - } - - size_t block_size = N / 4; - - if (argc > 2) - { - block_size = atoi(argv[2]); - fprintf(stderr, "block_size = %zu\n", block_size); - } - - // Do this lazily ? - cuda_safe_call(cublasCreate(&cublas_handle)); - - matrix A(N); - A.fill([&](int row, int col) { - return (1.0 / (row + col + 1) + (row == col ? 0.1 : 0.0)); - }); - - vector B(N, block_size); - vector X(N, block_size); - - B.fill([&](int /*unused*/) { - return 1.0; - }); - - X.fill([&](int /*unused*/) { - return 0.0; - }); - - cg(A, X, B); - - ctx.finalize(); -} diff --git a/cccl_upstream/cudax/examples/stf/linear_algebra/cg_solver.cuh b/cccl_upstream/cudax/examples/stf/linear_algebra/cg_solver.cuh deleted file mode 100644 index d8667735..00000000 --- a/cccl_upstream/cudax/examples/stf/linear_algebra/cg_solver.cuh +++ /dev/null @@ -1,189 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#pragma once -/** - * @file - * @brief Sparse conjugate gradient algorithm - */ - -#include - -#include "dot.cuh" - -using namespace cuda::experimental::stf; - -#if !_CCCL_CTK_BELOW(12, 4) -template -void cg_solver(ctx_t& ctx, csr_matrix& A, vector_t& X, vector_t& B, double cg_tol = 1e-10, size_t max_cg = 1000) -{ - // Initial guess X = 0 (better for Newton corrections) - ctx.parallel_for(X.shape(), X.write()).set_symbol("init_guess")->*[] _CCCL_DEVICE(size_t i, auto dX) { - dX(i) = 0.0; - }; - - // Residual R initialized to B - auto R = ctx.logical_data(B.shape()).set_symbol("R"); - ctx.parallel_for(R.shape(), R.write(), B.read()).set_symbol("R=B")->*[] _CCCL_DEVICE(size_t i, auto dR, auto dB) { - dR(i) = dB(i); - }; - - // R = R - A*X - auto Ax = ctx.logical_data(X.shape()).set_symbol("Ax"); - SPMV(ctx, A, X, Ax); - ctx.parallel_for(R.shape(), R.rw(), Ax.read()).set_symbol("R -= Ax")->*[] _CCCL_DEVICE(size_t i, auto dR, auto dAx) { - dR(i) -= dAx(i); - }; - - // P = R; - auto P = ctx.logical_data(R.shape()).set_symbol("P"); - ctx.parallel_for(P.shape(), P.write(), R.read()).set_symbol("P=R")->*[] _CCCL_DEVICE(size_t i, auto dP, auto dR) { - dP(i) = dR(i); - }; - - // RSOLD = R'*R - auto rsold = ctx.logical_data(shape_of>()).set_symbol("rsold"); - DOT(ctx, R, R, rsold); - - // CG iteration counter - auto cg_iter = ctx.logical_data(shape_of>()).set_symbol("cg_iter"); - ctx.parallel_for(box(1), cg_iter.write()).set_symbol("init_cg_iter")->*[] _CCCL_DEVICE(size_t i, auto diter) { - *diter = 0; - }; - - { - auto while_guard = ctx.while_graph_scope(); - - // Ap = A*P - auto Ap = ctx.logical_data(P.shape()).set_symbol("Ap"); - SPMV(ctx, A, P, Ap); - - // We don't compute alpha explicitly - // alpha = rsold / (p' * Ap); - auto pAp = ctx.logical_data(shape_of>()).set_symbol("pAp"); - DOT(ctx, P, Ap, pAp); - - // x = x + alpha * p; - ctx.parallel_for(X.shape(), X.rw(), rsold.read(), pAp.read(), P.read()).set_symbol("X+=alpha*P") - ->*[] _CCCL_DEVICE(size_t i, auto dX, auto drsold, auto dpAp, auto dP) { - T alpha = (*drsold / *dpAp); - dX(i) += alpha * dP(i); - }; - - // r = r - alpha * Ap; - ctx.parallel_for(R.shape(), R.rw(), rsold.read(), pAp.read(), Ap.read()).set_symbol("R-=alpha*Ap") - ->*[] _CCCL_DEVICE(size_t i, auto dR, auto drsold, auto dpAp, auto dAp) { - T alpha = (*drsold / *dpAp); - dR(i) -= alpha * dAp(i); - }; - - // rsnew = r' * r; - auto rsnew = ctx.logical_data(shape_of>()).set_symbol("rsnew"); - DOT(ctx, R, R, rsnew); - - while_guard.update_cond(rsnew.read(), cg_iter.rw())->*[cg_tol, max_cg] __device__(auto drsnew, auto diter) { - (*diter)++; // increment iteration counter - bool converged = (*drsnew < cg_tol * cg_tol); - // printf("CG iter %d: RES %e (tol=%e)\n", *diter, sqrt(*drsnew), cg_tol); - return !converged && (*diter < max_cg); - }; - - // p = r + (rsnew / rsold) * p; - ctx.parallel_for(P.shape(), P.rw(), R.read(), rsnew.read(), rsold.read()).set_symbol("P=r+(rsnew/rsold)*P") - ->*[] _CCCL_DEVICE(size_t i, auto dP, auto dR, auto drsnew, auto drsold) { - dP(i) = dR(i) + (*drsnew / *drsold) * dP(i); - }; - - // update old residual - ctx.parallel_for(box(1), rsold.write(), rsnew.read()).set_symbol("update_rsold") - ->*[] _CCCL_DEVICE(size_t i, auto drsold, auto drsnew) { - *drsold = *drsnew; - }; - } -} - -template -void cg_solver_no_while( - ctx_t& ctx, csr_matrix& A, vector_t& X, vector_t& B, double cg_tol = 1e-10, size_t max_cg = 1000) -{ - // Initial guess X = 0 (better for Newton corrections) - ctx.parallel_for(X.shape(), X.write()).set_symbol("init_guess")->*[] _CCCL_DEVICE(size_t i, auto dX) { - dX(i) = 0.0; - }; - - // Residual R initialized to B - auto R = ctx.logical_data(B.shape()).set_symbol("R"); - ctx.parallel_for(R.shape(), R.write(), B.read()).set_symbol("R=B")->*[] _CCCL_DEVICE(size_t i, auto dR, auto dB) { - dR(i) = dB(i); - }; - - // R = R - A*X - auto Ax = ctx.logical_data(X.shape()).set_symbol("Ax"); - SPMV(ctx, A, X, Ax); - ctx.parallel_for(R.shape(), R.rw(), Ax.read()).set_symbol("R -= Ax")->*[] _CCCL_DEVICE(size_t i, auto dR, auto dAx) { - dR(i) -= dAx(i); - }; - - // P = R; - auto P = ctx.logical_data(R.shape()).set_symbol("P"); - ctx.parallel_for(P.shape(), P.write(), R.read()).set_symbol("P=R")->*[] _CCCL_DEVICE(size_t i, auto dP, auto dR) { - dP(i) = dR(i); - }; - - // RSOLD = R'*R - auto rsold = ctx.logical_data(shape_of>()).set_symbol("rsold"); - DOT(ctx, R, R, rsold); - - size_t iter = 0; - auto rsnew = ctx.logical_data(shape_of>()).set_symbol("rsnew"); - - do - { - // Ap = A*P - auto Ap = ctx.logical_data(P.shape()).set_symbol("Ap"); - SPMV(ctx, A, P, Ap); - - // We don't compute alpha explicitly - // alpha = rsold / (p' * Ap); - auto pAp = ctx.logical_data(shape_of>()).set_symbol("pAp"); - DOT(ctx, P, Ap, pAp); - - // x = x + alpha * p; - ctx.parallel_for(X.shape(), X.rw(), rsold.read(), pAp.read(), P.read()).set_symbol("X+=alpha*P") - ->*[] _CCCL_DEVICE(size_t i, auto dX, auto drsold, auto dpAp, auto dP) { - T alpha = (*drsold / *dpAp); - dX(i) += alpha * dP(i); - }; - - // r = r - alpha * Ap; - ctx.parallel_for(R.shape(), R.rw(), rsold.read(), pAp.read(), Ap.read()).set_symbol("R-=alpha*Ap") - ->*[] _CCCL_DEVICE(size_t i, auto dR, auto drsold, auto dpAp, auto dAp) { - T alpha = (*drsold / *dpAp); - dR(i) -= alpha * dAp(i); - }; - - // rsnew = r' * r; - DOT(ctx, R, R, rsnew); - - // p = r + (rsnew / rsold) * p; - ctx.parallel_for(P.shape(), P.rw(), R.read(), rsnew.read(), rsold.read()).set_symbol("P=r+(rsnew/rsold)*P") - ->*[] _CCCL_DEVICE(size_t i, auto dP, auto dR, auto drsnew, auto drsold) { - dP(i) = dR(i) + (*drsnew / *drsold) * dP(i); - }; - - // update old residual - ctx.parallel_for(box(1), rsold.write(), rsnew.read()).set_symbol("update_rsold") - ->*[] _CCCL_DEVICE(size_t i, auto drsold, auto drsnew) { - *drsold = *drsnew; - }; - } while ((++iter < max_cg) && (ctx.wait(rsnew) > cg_tol * cg_tol)); -} - -#endif diff --git a/cccl_upstream/cudax/examples/stf/linear_algebra/dot.cuh b/cccl_upstream/cudax/examples/stf/linear_algebra/dot.cuh deleted file mode 100644 index cafeebc6..00000000 --- a/cccl_upstream/cudax/examples/stf/linear_algebra/dot.cuh +++ /dev/null @@ -1,70 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#pragma once - -//! \file -//! \brief DOT algorithm - -#include - -using namespace cuda::experimental::stf; - -template -using vector_t = stackable_logical_data>; - -template -using scalar_t = stackable_logical_data>; - -template -struct csr_matrix -{ - csr_matrix(stackable_logical_data> _val_handle, - stackable_logical_data> _row_handle, - stackable_logical_data> _col_handle) - : val_handle(mv(_val_handle)) - , row_handle(mv(_row_handle)) - , col_handle(mv(_col_handle)) - {} - - /* Description of the CSR */ - mutable stackable_logical_data> val_handle; - mutable stackable_logical_data> row_handle; - mutable stackable_logical_data> col_handle; -}; - -// Note that a and b might be the same logical data -template -void DOT(ctx_t& ctx, vector_t& a, vector_t& b, scalar_t& res) -{ - ctx.parallel_for(a.shape(), a.read(), b.read(), res.reduce(reducer::sum{})).set_symbol("DOT")->* - [] __device__(size_t i, auto da, auto db, T& dres) { - dres += da(i) * db(i); - }; -}; - -template -void SPMV(ctx_t& ctx, csr_matrix& a, vector_t& x, vector_t& y) -{ - ctx.parallel_for(y.shape(), a.val_handle.read(), a.col_handle.read(), a.row_handle.read(), x.read(), y.write()) - .set_symbol("SPMV") - ->*[] _CCCL_DEVICE(size_t row, auto da_val, auto da_col, auto da_row, auto dx, auto dy) { - int row_start = da_row(row); - int row_end = da_row(row + 1); - - double sum = 0.0; - for (int elt = row_start; elt < row_end; elt++) - { - sum += da_val(elt) * dx(da_col(elt)); - } - - dy(row) = sum; - }; -} diff --git a/cccl_upstream/cudax/examples/stf/linear_algebra/newton_solver.cuh b/cccl_upstream/cudax/examples/stf/linear_algebra/newton_solver.cuh deleted file mode 100644 index 8e693c89..00000000 --- a/cccl_upstream/cudax/examples/stf/linear_algebra/newton_solver.cuh +++ /dev/null @@ -1,172 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Generic Newton Solver - */ - -#include - -#include "cg_solver.cuh" -#include "dot.cuh" - -using namespace cuda::experimental::stf; - -/** - * Generic Newton solver for nonlinear systems F(x) = 0 - * - * @tparam ctx_t STF context type - * @tparam ResidualCallback Callback to compute residual F(x) - * @tparam JacobianCallback Callback to assemble Jacobian J = ∂F/∂x - * - * The callbacks must be callable with these signatures: - * - ResidualCallback: void fn(ctx_t&, const vector_t& x, const vector_t& x_prev, vector_t& - * residual) - * - JacobianCallback: void fn(ctx_t&, const vector_t& x, vector_t& jacobian_values) - */ -template -void newton_solver( - ctx_t& ctx, - vector_t& U, - vector_t& csr_values, - const vector_t& csr_row_offsets, - const vector_t& csr_col_ind, - ResidualCallback compute_residual_fn, - JacobianCallback assemble_jacobian_fn, - size_t max_newton = 20, - double newton_tol = 1e-10, - size_t max_cg = 100) -{ - auto U_prev = ctx.logical_data(U.shape()).set_symbol("U_prev"); - - ctx.parallel_for(U.shape(), U_prev.write(), U.read()).set_symbol("init_guess") - ->*[] __device__(size_t i, auto dU_prev, auto dU) { - dU_prev(i) = dU(i); - }; - - auto newton_norm2 = ctx.logical_data(shape_of>()).set_symbol("newton_norm2"); - auto newton_iter = ctx.logical_data(shape_of>()).set_symbol("newton_iter"); - ctx.parallel_for(box(1), newton_iter.write()).set_symbol("init_newton_iter")->*[] _CCCL_DEVICE(size_t i, auto diter) { - *diter = 0; - }; - - { - auto while_guard = ctx.while_graph_scope(); - - auto residual = ctx.logical_data(U.shape()).set_symbol("residual"); - auto delta = ctx.logical_data(U.shape()).set_symbol("delta"); - - // Compute residual F(U) - compute_residual_fn(ctx, U, U_prev, residual); - - // Compute Newton residual norm for convergence check - DOT(ctx, residual, residual, newton_norm2); - - // Assemble Jacobian J = ∂F/∂U - assemble_jacobian_fn(ctx, U, csr_values); - - auto rhs = ctx.logical_data(U.shape()).set_symbol("rhs"); - - // Set up RHS: rhs = -F(U) - ctx.parallel_for(rhs.shape(), rhs.write(), residual.read()).set_symbol("rhs = -residual") - ->*[] __device__(size_t i, auto drhs, auto dresidual) { - drhs(i) = -dresidual(i); - }; - - csr_matrix A(csr_values, csr_row_offsets, csr_col_ind); - - // Solve linear system: J * delta = -F(U) - double cg_tol = 1e-8; - cg_solver(ctx, A, delta, rhs, cg_tol, max_cg); - - // Newton update: U = U + delta (no special boundary handling needed) - ctx.parallel_for(U.shape(), U.rw(), delta.read()).set_symbol("newton_update") - ->*[] __device__(size_t i, auto dU, auto ddelta) { - dU(i) += ddelta(i); - }; - - while_guard.update_cond(newton_norm2.read(), newton_iter.rw()) - ->*[newton_tol, max_newton] __device__(auto dnorm2, auto diter) { - (*diter)++; // increment iteration counter - bool converged = (*dnorm2 < newton_tol * newton_tol); - return !converged && (*diter < max_newton); - }; - } -} - -template -void newton_solver_no_while( - ctx_t& ctx, - vector_t& U, - vector_t& csr_values, - const vector_t& csr_row_offsets, - const vector_t& csr_col_ind, - ResidualCallback compute_residual_fn, - JacobianCallback assemble_jacobian_fn, - bool cg_use_while = false, - size_t max_newton = 20, - double newton_tol = 1e-10, - size_t max_cg = 100) -{ - auto U_prev = ctx.logical_data(U.shape()).set_symbol("U_prev"); - - ctx.parallel_for(U.shape(), U_prev.write(), U.read()).set_symbol("init_guess") - ->*[] __device__(size_t i, auto dU_prev, auto dU) { - dU_prev(i) = dU(i); - }; - - auto newton_norm2 = ctx.logical_data(shape_of>()).set_symbol("newton_norm2"); - - size_t iter = 0; - do - { - auto residual = ctx.logical_data(U.shape()).set_symbol("residual"); - auto delta = ctx.logical_data(U.shape()).set_symbol("delta"); - - // Compute residual F(U) - compute_residual_fn(ctx, U, U_prev, residual); - - // Compute Newton residual norm for convergence check - DOT(ctx, residual, residual, newton_norm2); - - // Assemble Jacobian J = ∂F/∂U - assemble_jacobian_fn(ctx, U, csr_values); - - auto rhs = ctx.logical_data(U.shape()).set_symbol("rhs"); - - // Set up RHS: rhs = -F(U) - ctx.parallel_for(rhs.shape(), rhs.write(), residual.read()).set_symbol("rhs = -residual") - ->*[] __device__(size_t i, auto drhs, auto dresidual) { - drhs(i) = -dresidual(i); - }; - - csr_matrix A(csr_values, csr_row_offsets, csr_col_ind); - - // Solve linear system: J * delta = -F(U) - double cg_tol = 1e-8; - if (cg_use_while) - { - // fprintf(stderr, "NEWTON NO WHILE, CG WHILE.\n"); - cg_solver(ctx, A, delta, rhs, cg_tol, max_cg); - } - else - { - // fprintf(stderr, "NEWTON NO WHILE, CG NO WHILE.\n"); - cg_solver_no_while(ctx, A, delta, rhs, cg_tol, max_cg); - } - - // Newton update: U = U + delta (no special boundary handling needed) - ctx.parallel_for(U.shape(), U.rw(), delta.read()).set_symbol("newton_update") - ->*[] __device__(size_t i, auto dU, auto ddelta) { - dU(i) += ddelta(i); - }; - } while ((++iter < max_newton) && ctx.wait(newton_norm2) > newton_tol * newton_tol); -} diff --git a/cccl_upstream/cudax/examples/stf/linear_algebra/strassen.cu b/cccl_upstream/cudax/examples/stf/linear_algebra/strassen.cu deleted file mode 100644 index b28167cf..00000000 --- a/cccl_upstream/cudax/examples/stf/linear_algebra/strassen.cu +++ /dev/null @@ -1,489 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Strassen matrix multiplication algorithm - * - * This demonstrates how CUDASTF helps combining many interdependent tasks and - * deal with temporary data. - */ - -#include - -static const size_t BLOCKSIZE = 1024; - -using namespace cuda::experimental::stf; - -using logical_matrix = logical_data>; - -inline size_t get_m(logical_matrix& s) -{ - return s.shape().extent(0); -} - -inline size_t get_n(logical_matrix& s) -{ - return s.shape().extent(1); -} - -// XXX global for the sake of simplicity, yet ... -static std::vector cublas_handle; - -cublasHandle_t get_cublas_handle() -{ - int dev; - cuda_safe_call(cudaGetDevice(&dev)); - return cublas_handle[dev]; -} - -// C = AB -void MULT_CLASSIC(context& ctx, logical_matrix& A, logical_matrix& B, logical_matrix& C) -{ - ctx.task(A.read(), B.read(), C.write()).set_symbol("MULT")->*[](cudaStream_t s, auto a, auto b, auto c) { - cuda_safe_call(cublasSetStream(get_cublas_handle(), s)); - - size_t N = a.extent(0); - - const double zero = 0.0; - const double one = 1.0; - cuda_safe_call(cublasDgemm( - get_cublas_handle(), - CUBLAS_OP_N, - CUBLAS_OP_N, - N, - N, - N, - &one, - a.data_handle(), - a.stride(1), - b.data_handle(), - b.stride(1), - &zero, - c.data_handle(), - c.stride(1))); - }; -} - -// A = A + alpha B -template -__global__ void add_kernel(int m, int n, T* A, int ld_A, T alpha, const T* B, int ld_B) -{ - for (int idx = threadIdx.x + blockIdx.x * blockDim.x; idx < n; idx += blockDim.x * gridDim.x) - { - for (int idy = threadIdx.y + blockIdx.y * blockDim.y; idy < m; idy += blockDim.y * gridDim.y) - { - A[idy + idx * ld_A] += alpha * B[idy + idx * ld_B]; - } - } -} - -// Compute A = A + B -template -void ADD(context& ctx, logical_matrix& A, T alpha, logical_matrix& B) -{ - ctx.task(A.rw(), B.read()).set_symbol("ADD")->*[&](cudaStream_t s, auto a, auto b) { - int m_A = a.extent(0); - int n_A = a.extent(1); - - int ld_A = a.stride(1); - int ld_B = b.stride(1); - - T* addr_A = a.data_handle(); - const T* addr_B = b.data_handle(); - - add_kernel<<<16, 16, 0, s>>>(m_A, n_A, addr_A, ld_A, alpha, addr_B, ld_B); - }; -} - -template -__global__ void copy_kernel(int m, int n, const T* src, int ld_src, T* dst, int ld_dst) -{ - for (int idx = threadIdx.x + blockIdx.x * blockDim.x; idx < n; idx += blockDim.x * gridDim.x) - { - for (int idy = threadIdx.y + blockIdx.y * blockDim.y; idy < m; idy += blockDim.y * gridDim.y) - { - dst[idy + idx * ld_dst] = src[idy + idx * ld_src]; - } - } -} - -// row and col = 0 or 1 -template -void COPY_TO_SUBMATRIX(context& ctx, logical_data>& A, logical_data>& subA, int row, int col) -{ - // To copy to a subset, this is a write only access, so that we did not need a valid copy for subA before ... - ctx.task(A.read(), subA.write()).set_symbol("COPY_TO")->*[&](cudaStream_t s, auto a, auto subA) { - int ld_A = a.stride(1); - int ld_subA = subA.stride(1); - int m_subA = subA.extent(0); - int n_subA = subA.extent(1); - T* addr_subA = subA.data_handle(); - const T* addr_A_base = a.data_handle(); - const T* addr_A = addr_A_base + row * m_subA + col * n_subA * ld_A; - - // subA = A_row,col - copy_kernel<<<16, 16, 0, s>>>(m_subA, n_subA, addr_A, ld_A, addr_subA, ld_subA); - }; -} - -template -void COPY_FROM_SUBMATRICES(context& ctx, logical_data>& A, logical_data> subA[2][2]) -{ - // To copy to a subset, this is a write only access, so that we did not need a valid copy for subA before ... - // When copying from a subset to the whole matrix, we need a RW because we only modify a part of the matrix - ctx.task(A.write(), subA[0][0].read(), subA[0][1].read(), subA[1][0].read(), subA[1][1].read()).set_symbol("COPY_FROM") - ->*[&](cudaStream_t s, auto a, auto a00, auto a01, auto a10, auto a11) { - int ld_A = a.stride(1); - T* addr_A_base = a.data_handle(); - - for (int col = 0; col < 2; col++) - { - for (int row = 0; row < 2; row++) - { - auto& subA = col == 0 ? (row == 0 ? a00 : a10) : (row == 0 ? a01 : a11); - int m_subA = subA.extent(0); - int n_subA = subA.extent(1); - int ld_subA = subA.stride(1); - const T* addr_subA = subA.data_handle(); - T* addr_A = addr_A_base + row * m_subA + col * n_subA * ld_A; - - // A_row,col= subA - copy_kernel<<<16, 16, 0, s>>>(m_subA, n_subA, addr_subA, ld_subA, addr_A, ld_A); - } - } - }; -} - -template -void COPY_MATRIX(context& ctx, logical_data>& dst, logical_data>& src) -{ - // This is a write only access, so that we did not need a valid copy for subA before ... - ctx.task(dst.write(), src.read()).set_symbol("COPY")->*[&](cudaStream_t s, auto d_dst, auto d_src) { - int ld_src = d_dst.stride(1); - int ld_dst = d_src.stride(1); - - auto m = d_src.extent(0); - assert(m == d_dst.extent(0)); - - auto n = d_src.extent(1); - assert(n == d_dst.extent(1)); - - const T* addr_src = d_src.data_handle(); - T* addr_dst = d_dst.data_handle(); - - copy_kernel<<<16, 16, 0, s>>>(m, n, addr_src, ld_src, addr_dst, ld_dst); - }; -} - -void MULT(context& ctx, logical_matrix& A, logical_matrix& B, logical_matrix& C); - -void MULT_REC_NAIVE(context& ctx, logical_matrix& A, logical_matrix& B, logical_matrix& C) -{ - logical_matrix subA[2][2], subB[2][2], subC[2][2]; - - size_t N = get_m(A); - - assert(get_m(A) == get_n(A)); - assert(get_m(B) == get_n(B)); - assert(get_m(C) == get_n(C)); - - assert(N % 2 == 0); - - size_t half_N = N / 2; - - // These are TMP data which don't have a valid copy yet - for (int col = 0; col < 2; col++) - { - for (int row = 0; row < 2; row++) - { - subA[row][col] = ctx.logical_data(shape_of>(half_N, half_N)); - subB[row][col] = ctx.logical_data(shape_of>(half_N, half_N)); - subC[row][col] = ctx.logical_data(shape_of>(half_N, half_N)); - - COPY_TO_SUBMATRIX(ctx, A, subA[row][col], row, col); - COPY_TO_SUBMATRIX(ctx, B, subB[row][col], row, col); - } - } - - for (int col = 0; col < 2; col++) - { - for (int row = 0; row < 2; row++) - { - for (int k = 0; k < 2; k++) - { - auto Ck = ctx.logical_data(shape_of>(half_N, half_N)); - MULT(ctx, subA[row][k], subB[k][col], Ck); - - ADD(ctx, subC[row][col], 1.0, Ck); - } - - // C_row,col = subC[row][col] - COPY_FROM_SUBMATRICES(ctx, C, subC); - } - } -} - -void MULT_STRASSEN(context& ctx, logical_matrix& A, logical_matrix& B, logical_matrix& C) -{ - /* - * STRASSEN ALGORITHM - * - * M1 = (A00 + A11)(B00 + B11) - * M2 = (A10 + A11)B00 - * M3 = A00(B01 - B11) - * M4 = A11(B10 - B00) - * M5 = (A00 + A01)B11 - * M6 = (A10 - A00)(B00 + B01) - * M7 = (A01 - A11)(B10 + B11) - * - * C00 = M1 + M4 - M5 + M7 - * C01 = M3 + M5 - * C10 = M2 + M4 - * C11 = M1 - M2 + M3 + M6 - * - */ - size_t N = get_m(A); - assert(N % 2 == 0); - size_t half_N = N / 2; - - logical_matrix subA[2][2], subB[2][2], subC[2][2]; - auto M1 = ctx.logical_data(shape_of>(half_N, half_N)); - auto M2 = ctx.logical_data(shape_of>(half_N, half_N)); - auto M3 = ctx.logical_data(shape_of>(half_N, half_N)); - auto M4 = ctx.logical_data(shape_of>(half_N, half_N)); - auto M5 = ctx.logical_data(shape_of>(half_N, half_N)); - auto M6 = ctx.logical_data(shape_of>(half_N, half_N)); - auto M7 = ctx.logical_data(shape_of>(half_N, half_N)); - - assert(get_m(A) == get_n(A)); - assert(get_m(B) == get_n(B)); - assert(get_m(C) == get_n(C)); - - // These are TMP data which don't have a valid copy yet - for (int col = 0; col < 2; col++) - { - for (int row = 0; row < 2; row++) - { - subA[row][col] = ctx.logical_data(shape_of>(half_N, half_N)); - subB[row][col] = ctx.logical_data(shape_of>(half_N, half_N)); - subC[row][col] = ctx.logical_data(shape_of>(half_N, half_N)); - - COPY_TO_SUBMATRIX(ctx, A, subA[row][col], row, col); - COPY_TO_SUBMATRIX(ctx, B, subB[row][col], row, col); - } - } - - // M1 = (A00 + A11)(B00 + B11) - { - auto left = ctx.logical_data(shape_of>(half_N, half_N)), - right = ctx.logical_data(shape_of>(half_N, half_N)); - - COPY_MATRIX(ctx, left, subA[0][0]); - ADD(ctx, left, 1.0, subA[1][1]); - - COPY_MATRIX(ctx, right, subB[0][0]); - ADD(ctx, right, 1.0, subB[1][1]); - - MULT(ctx, left, right, M1); - } - - // M2 = (A10 + A11)B00 - { - auto left = ctx.logical_data(shape_of>(half_N, half_N)); - - COPY_MATRIX(ctx, left, subA[1][0]); - ADD(ctx, left, 1.0, subA[1][1]); - - MULT(ctx, left, subB[0][0], M2); - } - - // M3 = A00(B01 - B11) - { - auto right = ctx.logical_data(shape_of>(half_N, half_N)); - - COPY_MATRIX(ctx, right, subB[0][1]); - ADD(ctx, right, -1.0, subB[1][1]); - - MULT(ctx, subA[0][0], right, M3); - } - - // M4 = A11(B10 - B00) - { - auto right = ctx.logical_data(shape_of>(half_N, half_N)); - - COPY_MATRIX(ctx, right, subB[1][0]); - ADD(ctx, right, -1.0, subB[0][0]); - - MULT(ctx, subA[1][1], right, M4); - } - - // M5 = (A00 + A01)B11 - { - auto left = ctx.logical_data(shape_of>(half_N, half_N)); - - COPY_MATRIX(ctx, left, subA[0][0]); - ADD(ctx, left, 1.0, subA[0][1]); - - MULT(ctx, left, subB[1][1], M5); - } - - // M6 = (A10 - A00)(B00 + B01) - { - auto left = ctx.logical_data(shape_of>(half_N, half_N)), - right = ctx.logical_data(shape_of>(half_N, half_N)); - - COPY_MATRIX(ctx, left, subA[1][0]); - ADD(ctx, left, -1.0, subA[1][1]); - - COPY_MATRIX(ctx, right, subB[0][0]); - ADD(ctx, right, 1.0, subB[0][1]); - - MULT(ctx, left, right, M6); - } - - // M7 = (A01 - A11)(B10 + B11) - { - auto left = ctx.logical_data(shape_of>(half_N, half_N)); - auto right = ctx.logical_data(shape_of>(half_N, half_N)); - - COPY_MATRIX(ctx, left, subA[0][1]); - ADD(ctx, left, -1.0, subA[1][1]); - - COPY_MATRIX(ctx, right, subB[1][0]); - ADD(ctx, right, 1.0, subB[1][1]); - - MULT(ctx, left, right, M7); - } - - // C00 = M1 + M4 - M5 + M7 - COPY_MATRIX(ctx, subC[0][0], M1); - ADD(ctx, subC[0][0], 1.0, M4); - ADD(ctx, subC[0][0], -1.0, M5); - ADD(ctx, subC[0][0], -1.0, M5); - ADD(ctx, subC[0][0], 1.0, M7); - - // C01 = M3 + M5 - COPY_MATRIX(ctx, subC[0][1], M3); - ADD(ctx, subC[0][1], 1.0, M5); - - // C10 = M2 + M4 - COPY_MATRIX(ctx, subC[1][0], M2); - ADD(ctx, subC[1][0], 1.0, M4); - - // C11 = M1 - M2 + M3 + M6 - COPY_MATRIX(ctx, subC[1][1], M1); - ADD(ctx, subC[1][1], -1.0, M2); - ADD(ctx, subC[1][1], 1.0, M3); - ADD(ctx, subC[1][1], 1.0, M6); - - // Write back subsets of C to C - COPY_FROM_SUBMATRICES(ctx, C, subC); -} - -void MULT(context& ctx, logical_matrix& A, logical_matrix& B, logical_matrix& C) -{ - size_t N = get_m(A); - - if (N <= BLOCKSIZE) - { - MULT_CLASSIC(ctx, A, B, C); - } - else - { - // MULT_REC_NAIVE(ctx, A, B, C); - MULT_STRASSEN(ctx, A, B, C); - } -} - -void strassen_test(context& ctx, size_t N) -{ - double* A = new double[N * N]; - double* B = new double[N * N]; - double* C = new double[N * N]; - - int ldA = N; - int ldB = N; - int ldC = N; - - cuda_safe_call(cudaHostRegister(A, N * N * sizeof(double), cudaHostRegisterPortable)); - cuda_safe_call(cudaHostRegister(B, N * N * sizeof(double), cudaHostRegisterPortable)); - cuda_safe_call(cudaHostRegister(C, N * N * sizeof(double), cudaHostRegisterPortable)); - - for (size_t col = 0; col < N; col++) - { - for (size_t row = 0; row < N; row++) - { - A[row + N * col] = 1.0; - B[row + N * col] = -1.0; - C[row + N * col] = 0.0; - } - } - - auto descA = ctx.logical_data(make_slice(A, std::tuple{N, N}, ldA)), - descB = ctx.logical_data(make_slice(B, std::tuple{N, N}, ldB)), - descC = ctx.logical_data(make_slice(C, std::tuple{N, N}, ldC)); - descA.set_symbol("A"); - descB.set_symbol("B"); - descC.set_symbol("C"); - - std::chrono::steady_clock::time_point start, stop; - - ctx.host_launch(descC.read())->*[&](auto /* ignored */) { - start = std::chrono::steady_clock::now(); - }; - - MULT(ctx, descA, descB, descC); - - ctx.host_launch(descC.read())->*[&](auto /* ignored */) { - stop = std::chrono::steady_clock::now(); - }; - - ctx.finalize(); - - std::chrono::duration duration = stop - start; - fprintf(stderr, "Elapsed: %.2lf ms\n", duration.count() * 1000.0); -} - -int main(int argc, char** argv) -{ - long N = 2 * BLOCKSIZE; - - if (argc > 1) - { - N = atoi(argv[1]); - } - - bool use_graphs = false; - if (argc > 2) - { - use_graphs = (atoi(argv[2]) > 0); - } - - // Set up CUBLAS - int ndevs; - cuda_safe_call(cudaGetDeviceCount(&ndevs)); - cublas_handle.resize(ndevs); - for (int d = 0; d < ndevs; d++) - { - cuda_safe_call(cudaSetDevice(d)); - cuda_safe_call(cublasCreate(&cublas_handle[d])); - } - - cuda_safe_call(cudaSetDevice(0)); - - context ctx; - if (use_graphs) - { - ctx = graph_ctx(); - } - - strassen_test(ctx, N); -} diff --git a/cccl_upstream/cudax/examples/stf/logical_gates_composition.cu b/cccl_upstream/cudax/examples/stf/logical_gates_composition.cu deleted file mode 100644 index 1b58836c..00000000 --- a/cccl_upstream/cudax/examples/stf/logical_gates_composition.cu +++ /dev/null @@ -1,73 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Composition of boolean operations applied on logical data - */ - -#include - -using namespace cuda::experimental::stf; - -// z = AND(x,y) -logical_data> AND(context& ctx, logical_data> x, logical_data> y) -{ - assert(x.shape().size() == y.shape().size()); - - auto z = ctx.logical_data(x.shape()); - - std::string symbol = "(" + x.get_symbol() + " & " + y.get_symbol() + ")"; - z.set_symbol(symbol); - - ctx.parallel_for(z.shape(), x.read(), y.read(), z.write()).set_symbol("AND")->* - [] __device__(size_t i, auto dx, auto dy, auto dz) { - dz(i) = dx(i) & dy(i); - }; - - return z; -} - -// y = NOT(x) -logical_data> NOT(context& ctx, logical_data> x) -{ - auto y = ctx.logical_data(x.shape()); - - std::string symbol = "( !" + x.get_symbol() + ")"; - y.set_symbol(symbol); - - ctx.parallel_for(y.shape(), x.read(), y.write()).set_symbol("NOT")->*[] __device__(size_t i, auto dx, auto dy) { - dy(i) = ~dx(i); - }; - - return y; -} - -int main() -{ - const size_t n = 12; - - int X[n], Y[n], Z[n]; - - context ctx; - - auto lX = ctx.logical_data(X); - auto lY = ctx.logical_data(Y); - auto lZ = ctx.logical_data(Z); - - lX.set_symbol("X"); - lY.set_symbol("Y"); - lZ.set_symbol("Z"); - - auto lB = AND(ctx, AND(ctx, lX, lY), AND(ctx, lX, lZ)); - auto lC = AND(ctx, NOT(ctx, AND(ctx, lB, NOT(ctx, lY))), lX); - - ctx.finalize(); -} diff --git a/cccl_upstream/cudax/examples/stf/mandelbrot.cu b/cccl_upstream/cudax/examples/stf/mandelbrot.cu deleted file mode 100644 index bcaa325b..00000000 --- a/cccl_upstream/cudax/examples/stf/mandelbrot.cu +++ /dev/null @@ -1,129 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief A transparent multi-GPU implementation of Mandelbrot fractal using parallel_for - */ - -#include - -#include -#include - -using namespace cuda::experimental::stf; - -int main(int argc, char** argv) -{ - context ctx; - - // Image dimensions - size_t width = 2000; - size_t height = 1000; - - // Complex plane boundaries - double xMin = -2.0; - double xMax = 1.0; - double yMin = -1.5; - double yMax = 1.5; - - // Maximum number of iterations - int maxIterations = 256; - - // Describe a 2D array of integers of size (width x height) - auto lbuffer = ctx.logical_data(shape_of>(width, height)); - - cudaEvent_t start, stop; - cuda_safe_call(cudaEventCreate(&start)); - cuda_safe_call(cudaEventCreate(&stop)); - - cuda_safe_call(cudaEventRecord(start, ctx.fence())); - - // Compute each pixel - ctx.parallel_for(blocked_partition(), exec_place::all_devices(), lbuffer.shape(), lbuffer.write()) - ->*[=] _CCCL_DEVICE(size_t x, size_t y, auto buffer) { - // Map pixel coordinates to complex plane - // c = cr + i ci - double cr = x * (xMax - xMin) / width + xMin; - double ci = y * (yMax - yMin) / height + yMin; - - // z = zr + i zi - double zr = 0.0; - double zi = 0.0; - - int iterations = 0; - - // Evaluate depth - while (zr * zr + zi * zi < 4 && iterations < maxIterations) - { - // compute : z = z * z + c; - // - // z = (zr + i zi) (zr + i zi) + cr + i ci - // z = zr zr - zi zi + 2 i zi zr + cr + i ci - // zr = (zr zr - zi zi + cr) - // zi = (2 zi zr + ci) - double zr_prev = zr; - double zi_prev = zi; - zr = zr_prev * zr_prev - zi_prev * zi_prev + cr; - zi = 2.0 * zr_prev * zi_prev + ci; - - iterations++; - } - - buffer(x, y) = iterations; - }; - - cuda_safe_call(cudaEventRecord(stop, ctx.fence())); - - if (argc > 1) - { - auto fileName = std::string(argv[1]); - - // Generate a PPM file from the buffer - ctx.host_launch(lbuffer.read())->*[&](auto buffer) { - std::ofstream imageFile(fileName, std::ios::binary); - if (!imageFile) - { - std::cerr << "Failed to create image file: " << fileName << '\n'; - return; - } - - imageFile << "P6\n"; - imageFile << width << " " << height << "\n"; - imageFile << "255\n"; - - for (size_t y = 0; y < height; y++) - { - for (size_t x = 0; x < width; x++) - { - int iterations = buffer(x, y); - // Convert iterations to RGB values - unsigned char r = (iterations % 8) * 32; - unsigned char g = (iterations % 16) * 16; - unsigned char b = (iterations % 32) * 8; - - // Write pixel data to file - imageFile << r << g << b; - } - } - - imageFile.close(); - std::cout << "Mandelbrot image generated and saved as " << fileName << '\n'; - }; - } - - ctx.finalize(); - - // Must call this first, see e.g. - // https://stackoverflow.com/questions/6551121/cuda-cudaeventelapsedtime-returns-device-not-ready-error - cuda_safe_call(cudaEventSynchronize(stop)); - - fprintf(stderr, "Mandelbrot took %.2f ms\n", cuda_try(start, stop)); -} diff --git a/cccl_upstream/cudax/examples/stf/parallel_for_2D.cu b/cccl_upstream/cudax/examples/stf/parallel_for_2D.cu deleted file mode 100644 index 3670a932..00000000 --- a/cccl_upstream/cudax/examples/stf/parallel_for_2D.cu +++ /dev/null @@ -1,74 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief An example using parallel_for on shapes with different dimensions - */ - -#include - -using namespace cuda::experimental::stf; - -__host__ __device__ double x0(size_t i, size_t j) -{ - return sin((double) (i - j)); -} - -__host__ __device__ double y0(size_t i, size_t j) -{ - return cos((double) (i + j)); -} - -int main() -{ - context ctx; - - const size_t N = 16; - double X[2 * N * 2 * N]; - double Y[N * N]; - - auto lx = ctx.logical_data(make_slice(&X[0], std::tuple{2 * N, 2 * N}, 2 * N)); - auto ly = ctx.logical_data(make_slice(&Y[0], std::tuple{N, N}, N)); - - ctx.parallel_for(lx.shape(), lx.write())->*[=] _CCCL_DEVICE(size_t i, size_t j, auto sx) { - sx(i, j) = x0(i, j); - }; - - ctx.parallel_for(ly.shape(), lx.read(), ly.write())->*[=] _CCCL_DEVICE(size_t i, size_t j, auto sx, auto sy) { - sy(i, j) = y0(i, j); - for (size_t ii = 0; ii < 2; ii++) - { - for (size_t jj = 0; jj < 2; jj++) - { - sy(i, j) += sx(2 * i + ii, 2 * j + jj); - } - } - }; - - ctx.parallel_for(exec_place::host(), ly.shape(), ly.read()) - ->*[=] __host__(size_t i, size_t j, slice sy) { - double expected = y0(i, j); - for (size_t ii = 0; ii < 2; ii++) - { - for (size_t jj = 0; jj < 2; jj++) - { - expected += x0(2 * i + ii, 2 * j + jj); - } - } - - if (fabs(sy(i, j) - expected) > 0.001) - { - printf("sy(%zu, %zu) %f expect %f\n", i, j, sy(i, j), expected); - } - }; - - ctx.finalize(); -} diff --git a/cccl_upstream/cudax/examples/stf/partitioned_axpy.cu b/cccl_upstream/cudax/examples/stf/partitioned_axpy.cu deleted file mode 100644 index bd867e0c..00000000 --- a/cccl_upstream/cudax/examples/stf/partitioned_axpy.cu +++ /dev/null @@ -1,143 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief AXPY over data distributed across the machine's devices with a - * structured partition specification - * - * The partition ("dimension 0, blocked over the grid of devices") is - * expressed once as a cute_partition. The same description is then used to: - * - * 1. EVALUATE the placement before committing any memory - * (evaluate_localized_placement: bytes per place, placement accuracy); - * 2. back a logical data with a composite data place, so STF tasks operate - * on memory whose pages physically live on the device that owns them; - * 3. perform a raw geometry-aware allocation (allocate_nd(data_dims, elemsize)) - * outside of any STF context. - * - * Each place computes its own blocked portion (the idiomatic grid-task - * pattern), so no cross-device access is required; peer/mempool access setup - * is handled by the places machinery itself. - */ - -#include - -#include -#include - -using namespace cuda::experimental::stf; - -__global__ void axpy(size_t start, size_t cnt, double a, const double* x, double* y) -{ - int tid = blockIdx.x * blockDim.x + threadIdx.x; - int nthreads = gridDim.x * blockDim.x; - - for (size_t i = tid; i < cnt; i += nthreads) - { - y[start + i] += a * x[start + i]; - } -} - -double X0(size_t i) -{ - return sin((double) i); -} - -double Y0(size_t i) -{ - return cos((double) i); -} - -int main() -{ - // The places machinery enumerates the devices and sets up peer/mempool - // access between them; on a single-GPU machine this is one place. - auto all_devs = exec_place::all_devices(); - const size_t nplaces = all_devs.get_dims().size(); - - const size_t N = 4 * 1024 * 1024; - - // "Dimension 0, blocked over grid axis 0" - the per-dimension specification - auto part = make_partition(dim4(N), partition_spec{blocked<0>}, all_devs.get_dims()); - - // 1. Score the mapping before allocating anything - auto stats = evaluate_localized_placement(all_devs, part, sizeof(double)); - printf("Placement over %zu place(s): %zu blocks in %zu allocations, accuracy %.1f%%\n", - nplaces, - stats.nblocks, - stats.nallocs, - 100.0 * stats.accuracy()); - for (const auto& entry : stats.bytes_per_place) - { - printf(" %s: %.2f MB\n", entry.first.c_str(), entry.second / (1024.0 * 1024.0)); - } - - // 2. Run STF tasks over logical data placed by the same policy - stream_ctx ctx; - - ::std::vector X(N), Y(N); - for (size_t i = 0; i < N; i++) - { - X[i] = X0(i); - Y[i] = Y0(i); - } - - auto lX = ctx.logical_data(&X[0], {N}); - auto lY = ctx.logical_data(&Y[0], {N}); - - const double alpha = 3.14; - - // The composite data place distributes instances across the grid with the - // classic blocked partitioner (the callback form of the same policy) - auto dist = data_place::composite(blocked_partition_custom<0>{}, all_devs); - - // One task over the grid; each place computes its own blocked chunk - auto t = ctx.task(all_devs, lX.read(dist), lY.rw(dist)); - t->*[&](auto, auto dX, auto dY) { - const size_t chunk = (N + nplaces - 1) / nplaces; - for (size_t i = 0; i < nplaces; i++) - { - const size_t start = i * chunk; - if (start >= N) - { - // With ceil-division chunks, trailing places may have no work - continue; - } - const size_t cnt = ::std::min(chunk, N - start); - auto active = t.activate_place(i); - axpy<<<128, 128, 0, t.get_stream(i)>>>(start, cnt, alpha, dX.data_handle(), dY.data_handle()); - } - }; - - ctx.finalize(); - - for (size_t i = 0; i < N; i++) - { - if (fabs(Y[i] - (Y0(i) + alpha * X0(i))) > 0.0001) - { - fprintf(stderr, "Verification FAILED at %zu\n", i); - return 1; - } - } - printf("STF task over composite-placed data: verified\n"); - - // 3. Raw geometry-aware allocation, no STF context involved - auto dp = ::cuda::experimental::places::make_composite_data_place(all_devs, part); - void* raw = dp.allocate_nd(dim4(N), sizeof(double)); - auto* d_buf = static_cast(raw); - cuda_safe_call(cudaMemset(d_buf, 0, N * sizeof(double))); - cuda_safe_call(cudaDeviceSynchronize()); - dp.deallocate(raw, N * sizeof(double)); - printf("Raw shaped allocation on the partitioned place: OK\n"); - - return 0; -} diff --git a/cccl_upstream/cudax/examples/stf/pi.cu b/cccl_upstream/cudax/examples/stf/pi.cu deleted file mode 100644 index 9543ff41..00000000 --- a/cccl_upstream/cudax/examples/stf/pi.cu +++ /dev/null @@ -1,52 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Approximate pi using Monte Carlo method - * - */ - -#include - -#include -#include - -using namespace cuda::experimental::stf; - -int main(int, char**) -{ - context ctx; - auto lsum = ctx.logical_data(shape_of>()); - - size_t N = 1000000; - - ctx.parallel_for(box(N), lsum.reduce(reducer::sum{}))->*[] __device__(size_t i, auto& sum) { - curandState local_state; - curand_init(1234, i, 0, &local_state); - double x = curand_uniform_double(&local_state); // Random x in [0, 1) - double y = curand_uniform_double(&local_state); // Random y in [0, 1) - // Count (x,y) coordinates which are within the unit circle - if (x * x + y * y <= 1.0) - { - sum++; - } - }; - - // We get the ratio of "shots" within the unit circle and the total number of - // "shots". The surface of the quarter of unit circle [0, 1) x [0, 1) is pi/4 - auto res = ctx.wait(lsum); - double pi_val = (4.0 * res) / N; - - ctx.finalize(); - - _CCCL_ASSERT(fabs(pi_val - 3.1415) < 0.1, "Invalid result"); -} diff --git a/cccl_upstream/cudax/examples/stf/scan.cu b/cccl_upstream/cudax/examples/stf/scan.cu deleted file mode 100644 index 22bffd55..00000000 --- a/cccl_upstream/cudax/examples/stf/scan.cu +++ /dev/null @@ -1,204 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief A parallel scan algorithm using CUB kernels - * - */ - -#include // or equivalently - -#include - -using namespace cuda::experimental::stf; - -__host__ __device__ double X0(int i) -{ - return sin((double) i); -} - -/** - * @brief Performs an inclusive scan on a logical data slice using CUB. - * - * This function determines the temporary device storage requirements for a scan, allocates - * temporary storage, and then performs the scan using the CUB library. The scan is performed - * in place, modifying the input `logical_data` slice. - * - * @tparam Ctx The context type for data management and task execution. - * @tparam T The data type of the elements in the `logical_data` slice. - * - * @param ctx Reference to the context object. - * @param ld Reference to the `logical_data` object containing the data slice. - * @param dp The `data_place` enum specifying where the data should reside (e.g., CPU, GPU). - */ -template -void scan(Ctx& ctx, logical_data>& ld, data_place dp) -{ - // Determine temporary device storage requirements - auto num_items = int(ld.shape().size()); - size_t tmp_size = 0; - cub::DeviceScan::InclusiveSum(nullptr, tmp_size, (T*) nullptr, (T*) nullptr, num_items); - - // fprintf(stderr, "SCAN %ld items TMP = %ld bytes\n", num_items, tmp_size); - - logical_data> ltmp = ctx.logical_data(shape_of>(tmp_size)).set_symbol("tmp"); - - ctx.task(ld.rw(mv(dp)), ltmp.write()).set_symbol("scan " + ld.get_symbol()) - ->*[=](cudaStream_t stream, auto d, auto tmp) mutable { - T* buffer = d.data_handle(); - cub::DeviceScan::InclusiveSum(tmp.data_handle(), tmp_size, buffer, buffer, num_items, stream); - }; -} - -int main(int argc, char** argv) -{ - stream_ctx ctx; - // graph_ctx ctx; - - // const size_t N = 128ULL*1024ULL*1024ULL; - size_t nmb = 128; - if (argc > 1) - { - nmb = atoi(argv[1]); - } - - int check = 0; - if (argc > 2) - { - check = atoi(argv[2]); - } - - const size_t N = nmb * 1024ULL * 1024ULL; - - const int ndevs = cuda_try(); - const size_t NBLOCKS = 2 * ndevs; - - size_t BLOCK_SIZE = (N + NBLOCKS - 1) / NBLOCKS; - - auto fixed_alloc = block_allocator(ctx, BLOCK_SIZE * sizeof(double)); - ctx.set_allocator(fixed_alloc); - - // dummy task to initialize the allocator XXX - { - auto ldummy = ctx.logical_data(shape_of>(NBLOCKS)).set_symbol("dummy"); - ctx.task(ldummy.write(data_place::managed()))->*[](cudaStream_t, auto) {}; - } - - std::vector X(N); - std::vector>> lX(NBLOCKS); - logical_data> laux; - - // If we were to register each part one by one, there could be pages which - // cross multiple parts, and the pinning operation would fail. - cuda_safe_call(cudaHostRegister(&X[0], N * sizeof(double), cudaHostRegisterPortable)); - - for (size_t b = 0; b < NBLOCKS; b++) - { - size_t start = b * BLOCK_SIZE; - size_t end = std::min(start + BLOCK_SIZE, N); - lX[b] = ctx.logical_data(&X[start], {end - start}).set_symbol("X_" + std::to_string(b)); - - // No need to move this back to the host if we do not check the result - if (!check) - { - lX[b].set_write_back(false); - } - } - - for (size_t b = 0; b < NBLOCKS; b++) - { - cuda_safe_call(cudaSetDevice(b % ndevs)); - size_t start = b * BLOCK_SIZE; - ctx.parallel_for(lX[b].shape(), lX[b].write())->*[=] _CCCL_DEVICE(size_t i, auto lx) { - lx(i) = X0(i + start); - }; - } - - cuda_safe_call(cudaStreamSynchronize(ctx.fence())); - - cudaEvent_t start, stop; - cuda_safe_call(cudaEventCreate(&start)); - cuda_safe_call(cudaEventCreate(&stop)); - cuda_safe_call(cudaEventRecord(start, ctx.fence())); - - for (size_t k = 0; k < 100; k++) - { - // Create an auxiliary temporary buffer and blank it - laux = ctx.logical_data(shape_of>(NBLOCKS)).set_symbol("aux"); - ctx.parallel_for(laux.shape(), laux.write(data_place::managed())).set_symbol("init_aux") - ->*[] _CCCL_DEVICE(size_t i, auto aux) { - aux(i) = 0.0; - }; - - // Scan each block - for (size_t b = 0; b < NBLOCKS; b++) - { - cuda_safe_call(cudaSetDevice(b % ndevs)); - scan(ctx, lX[b], data_place::device(b % ndevs)); - } - - for (size_t b = 0; b < NBLOCKS; b++) - { - // cuda_safe_call(cudaSetDevice(b % ndevs)); - - ctx.parallel_for(exec_place::device(0), - box({b, b + 1}), - lX[b].read(data_place::device(b % ndevs)), - laux.rw(data_place::managed())) - .set_symbol("store sum X_" + std::to_string(b)) - ->*[] _CCCL_DEVICE(size_t ind, auto Xb, auto aux) { - aux(ind) = Xb(Xb.extent(0) - 1); - }; - } - - // Prefix sum of the per-block sums - scan(ctx, laux, data_place::managed()); - - // Add partial sum of Xi to X(i+1) - for (size_t b = 1; b < NBLOCKS; b++) - { - cuda_safe_call(cudaSetDevice(b % ndevs)); - ctx.parallel_for(lX[b].shape(), lX[b].rw(), laux.read(data_place::managed())) - .set_symbol("add X_" + std::to_string(b)) - ->*[=] _CCCL_DEVICE(size_t i, auto Xb, auto aux) { - Xb(i) += aux(b - 1); - }; - } - } - - cuda_safe_call(cudaEventRecord(stop, ctx.fence())); - - ctx.finalize(); - - float ms = 0; - cuda_safe_call(cudaEventElapsedTime(&ms, start, stop)); - - fprintf(stdout, "%zu %f ms\n", N / 1024 / 1024, ms); - - if (check) - { -#if 0 - for (size_t i = 0; i < N; i++) { - EXPECT(fabs(X[i] - expected_result[i]) < 0.00001); - } -#endif - -#if 1 - fprintf(stderr, "Checking result ...\n"); - EXPECT(fabs(X[0] - X0(0)) < 0.00001); - for (size_t i = 0; i < N; i++) - { - EXPECT(fabs(X[i] - X[i - 1] - X0(i)) < 0.00001); - } - } -#endif -} diff --git a/cccl_upstream/cudax/examples/stf/sqrt_newton_stackable.cu b/cccl_upstream/cudax/examples/stf/sqrt_newton_stackable.cu deleted file mode 100644 index 50ba75f9..00000000 --- a/cccl_upstream/cudax/examples/stf/sqrt_newton_stackable.cu +++ /dev/null @@ -1,80 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Compute square roots via Newton's method using while_graph_scope - * - * This is a minimal example of an iterative solver with convergence - * checking in a stackable context. Each iteration applies the - * Babylonian step x <- (x + S/x) / 2 and reduces the maximum - * absolute change across all elements. The while loop exits once - * the change drops below a tolerance. - */ - -#include - -using namespace cuda::experimental::stf; - -int main() -{ -#if _CCCL_CTK_BELOW(12, 4) - fprintf(stderr, "Waiving example: while_graph_scope requires CUDA 12.4+.\n"); - return 0; -#else - stackable_ctx ctx; - - constexpr size_t N = 1024; - constexpr double tol = 1e-12; - - ::std::vector host_S(N); - ::std::vector host_X(N); - - for (size_t i = 0; i < N; i++) - { - host_S[i] = 1.0 + static_cast(i); - host_X[i] = host_S[i]; // initial guess x0 = S - } - - auto lS = ctx.logical_data(make_slice(host_S.data(), N)).set_symbol("S"); - lS.set_read_only(); - - auto lX = ctx.logical_data(make_slice(host_X.data(), N)).set_symbol("X"); - auto lmax_err = ctx.logical_data(shape_of>()).set_symbol("max_err"); - - { - auto while_guard = ctx.while_graph_scope(); - - // Babylonian step: x = (x + S/x) / 2, reduce max |change| - ctx.parallel_for(box(N), lX.rw(), lS.read(), lmax_err.reduce(reducer::maxval{})) - ->*[] __device__(size_t i, auto x, auto s, auto& max_err) { - double x_old = x(i); - double x_new = 0.5 * (x_old + s(i) / x_old); - x(i) = x_new; - max_err = fabs(x_new - x_old); - }; - - while_guard.update_cond(lmax_err.read())->*[tol] __device__(auto max_err) { - return (*max_err > tol); - }; - } - - ctx.finalize(); - - for (size_t i = 0; i < N; i++) - { - double expected = sqrt(1.0 + static_cast(i)); - EXPECT(fabs(host_X[i] - expected) < 1e-8); - } - - return 0; -#endif -} diff --git a/cccl_upstream/cudax/examples/stf/standalone-launches.cu b/cccl_upstream/cudax/examples/stf/standalone-launches.cu deleted file mode 100644 index 83ad6fc2..00000000 --- a/cccl_upstream/cudax/examples/stf/standalone-launches.cu +++ /dev/null @@ -1,77 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief This test illustrates how we can use multiple reserved::launch in a single task on different pieces of data - */ - -#include - -using namespace cuda::experimental::stf; - -int X0(int i) -{ - return i * i + 12; -} - -int main() -{ - stream_ctx ctx; - - const int N = 16; - int X[N], Y[N], Z[N]; - - for (size_t ind = 0; ind < N; ind++) - { - X[ind] = X0(ind); - Y[ind] = 0; - Z[ind] = 0; - } - - auto handle_X = ctx.logical_data(X, {N}); - auto handle_Y = ctx.logical_data(Y, {N}); - auto handle_Z = ctx.logical_data(Z, {N}); - - ctx.task(handle_X.read(), handle_Y.write(), handle_Z.write()) - ->*[](cudaStream_t s, slice x, slice y, slice z) { - std::vector streams; - streams.push_back(s); - auto spec = par(1024); - reserved::launch(spec, exec_place::current_device(), streams, std::tuple{x, y}) - ->*[] _CCCL_DEVICE(auto t, slice x, slice y) { - size_t tid = t.rank(); - size_t nthreads = t.size(); - for (size_t ind = tid; ind < N; ind += nthreads) - { - y(ind) = 2 * x(ind); - } - }; - - reserved::launch(spec, exec_place::current_device(), streams, std::tuple{y, z}) - ->*[] _CCCL_DEVICE(auto t, slice y, slice z) { - size_t tid = t.rank(); - size_t nthreads = t.size(); - for (size_t ind = tid; ind < N; ind += nthreads) - { - z(ind) = 3 * y(ind); - } - }; - }; - - ctx.finalize(); - - for (size_t ind = 0; ind < N; ind++) - { - assert(Y[ind] == 2 * X[ind]); - assert(Z[ind] == 3 * Y[ind]); - } -} diff --git a/cccl_upstream/cudax/examples/stf/thrust_zip_iterator.cu b/cccl_upstream/cudax/examples/stf/thrust_zip_iterator.cu deleted file mode 100644 index 76558619..00000000 --- a/cccl_upstream/cudax/examples/stf/thrust_zip_iterator.cu +++ /dev/null @@ -1,135 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief This example illustrates how we can convert Thrust iterators to - * logical data, and how to create thrust iterators from data instances in a - * task. - */ - -#include -#include -#include -#include -#include - -#include - -#include - -using namespace cuda::experimental::stf; - -// Functor to apply the transformation -struct my_transform_functor -{ - __host__ __device__ int operator()(const cuda::std::tuple& t) const - { - int a = cuda::std::get<0>(t); - char b = cuda::std::get<1>(t); - return a + static_cast(b); // Example operation - } -}; - -/* - * How to use CUDASTF to manipulate data originally created using Thrust - */ -template -void thrust_algorithm(context& ctx, ZippedIt& first, ZippedIt& last, OutIt& output, data_place data_location) -{ - /* - * Interpret Thrust data structures as logical data - */ - size_t num_elements = cuda::std::distance(first, last); - - // Extract underlying iterators from the zip iterator - auto itA = cuda::std::get<0>(first.get_iterator_tuple()); - int* A = thrust::raw_pointer_cast(&(*itA)); - - auto itB = cuda::std::get<1>(first.get_iterator_tuple()); - char* B = thrust::raw_pointer_cast(&(*itB)); - - int* C = thrust::raw_pointer_cast(output.data()); - - auto lA = ctx.logical_data(make_slice(A, num_elements), data_location); - auto lB = ctx.logical_data(make_slice(B, num_elements), data_location); - auto lC = ctx.logical_data(make_slice(C, num_elements), data_location); - - /* Important : result C will only be valid once we finalize the context or introduce a task fence ! */ - ctx.task(lA.read(), lB.read(), lC.write())->*[](cudaStream_t stream, auto dA, auto dB, auto dC) { - // Reconstruct a zipped iterator from the data instances passed to the lambda function - size_t num_elements = dA.size(); - auto dfirst = thrust::make_zip_iterator(cuda::std::tuple(dA.data_handle(), dB.data_handle())); - auto dlast = dfirst + num_elements; - - // Create a device pointer from the raw pointer - thrust::device_ptr dout = thrust::device_pointer_cast(dC.data_handle()); - - thrust::transform(thrust::cuda::par_nosync.on(stream), dfirst, dlast, dout, my_transform_functor()); - }; -} - -int main() -{ - context ctx; - - /* - * First create device vectors and zipped them - */ - thrust::device_vector A(3); - thrust::device_vector B(3); - thrust::device_vector C(3); - - A[0] = 10; - A[1] = 20; - A[2] = 30; - B[0] = 'x'; - B[1] = 'y'; - B[2] = 'z'; - - auto first = thrust::make_zip_iterator(cuda::std::tuple(A.begin(), B.begin())); - auto last = thrust::make_zip_iterator(cuda::std::tuple(A.end(), B.end())); - - thrust_algorithm(ctx, first, last, C, data_place::current_device()); - - /* - * Use host data, and rely on CUDASTF for transfers - */ - - thrust::host_vector hA(3); - thrust::host_vector hB(3); - thrust::host_vector hC(3); - - hA[0] = 10; - hA[1] = 20; - hA[2] = 30; - hB[0] = 'x'; - hB[1] = 'y'; - hB[2] = 'z'; - - auto hfirst = thrust::make_zip_iterator(cuda::std::tuple(hA.begin(), hB.begin())); - auto hlast = thrust::make_zip_iterator(cuda::std::tuple(hA.end(), hB.end())); - - thrust_algorithm(ctx, hfirst, hlast, hC, data_place::host()); - - /* Before this, we cannot assume that the Thrust algorithms have been - * performed and/or that the results have been written back to their original - * location. */ - ctx.finalize(); - - // Check results - for (size_t i = 0; i < 3; i++) - { - EXPECT(C[i] == (A[i] + static_cast(B[i]))); - EXPECT(hC[i] == (hA[i] + static_cast(hB[i]))); - } - - return 0; -} diff --git a/cccl_upstream/cudax/examples/stf/void_data_interface.cu b/cccl_upstream/cudax/examples/stf/void_data_interface.cu deleted file mode 100644 index bf429f23..00000000 --- a/cccl_upstream/cudax/examples/stf/void_data_interface.cu +++ /dev/null @@ -1,57 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * - * @brief Illustrate how to use the void data interface - * - */ - -#include - -using namespace cuda::experimental::stf; - -__global__ void dummy_kernel() {} - -int main() -{ - context ctx; - - auto token = ctx.token(); - ctx.task(token.write())->*[](cudaStream_t) { - - }; - - void_interface sync; - auto token2 = ctx.logical_data(sync); - - auto token3 = ctx.token(); - ctx.task(token2.write(), token.read())->*[](cudaStream_t) { - - }; - - // Do not pass useless arguments by removing void_interface arguments - // Note that the rw() access is possible even if there was no prior write() - // or actual underlying data. - ctx.task(token3.rw(), token.read())->*[](cudaStream_t) { - - }; - - ctx.cuda_kernel(token3.rw())->*[]() { - return cuda_kernel_desc{dummy_kernel, 16, 128, 0}; - }; - - EXPECT(token.is_void_interface()); - EXPECT(token2.is_void_interface()); - EXPECT(token3.is_void_interface()); - - ctx.finalize(); -} diff --git a/cccl_upstream/cudax/examples/stf/word_count.cu b/cccl_upstream/cudax/examples/stf/word_count.cu deleted file mode 100644 index 3d76c194..00000000 --- a/cccl_upstream/cudax/examples/stf/word_count.cu +++ /dev/null @@ -1,91 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Counting words in a text using a launch kernel - */ - -#include - -using namespace cuda::experimental::stf; - -// determines whether the character is alphabetical -__host__ __device__ bool is_alpha(const char c) -{ - return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); -} - -int main() -{ - // Paragraph from 'The Raven' by Edgar Allan Poe - // http://en.wikipedia.org/wiki/The_Raven - const char raw_input[] = - " But the raven, sitting lonely on the placid bust, spoke only,\n" - " That one word, as if his soul in that one word he did outpour.\n" - " Nothing further then he uttered - not a feather then he fluttered -\n" - " Till I scarcely more than muttered `Other friends have flown before -\n" - " On the morrow he will leave me, as my hopes have flown before.'\n" - " Then the bird said, `Nevermore.'\n"; - - context ctx; - - auto ltext = ctx.logical_data(const_cast(&raw_input[0]), {sizeof(raw_input)}); - - int cnt = 0; - auto lcnt = ctx.logical_data(&cnt, {1}); - - auto number_devices = 2; - auto all_devs = exec_place::repeat(exec_place::device(0), number_devices); - - auto spec = par(con(128)); - - ctx.launch(spec, all_devs, ltext.read(), lcnt.rw())->*[] _CCCL_DEVICE(auto th, auto text, auto cnt) { - int local_cnt = 0; - for (size_t i = th.rank(); i < text.size() - 1; i += th.size()) - { - /* If the thread encounters the beginning of a new word, increment - * its local counter */ - if (!is_alpha(text(i)) && is_alpha(text(i + 1))) - { - local_cnt++; - } - } - - // Get a piece of shared memory, and zero it - __shared__ int block_cnt; - block_cnt = 0; - th.inner().sync(); - - // In every block, partial sums are gathered, and added to the result - // by the first thread of the block. - atomicAdd(&block_cnt, local_cnt); - th.inner().sync(); - - if (th.inner().rank() == 0) - { - atomicAdd(&cnt(0), block_cnt); - } - }; - - ctx.finalize(); - - int ref_cnt = 0; - for (size_t i = 0; i < sizeof(raw_input) - 1; i++) - { - if (!is_alpha(raw_input[i]) && is_alpha(raw_input[i + 1])) - { - ref_cnt++; - } - } - - // fprintf(stderr, "Result : found %d words (expected %d)\n", cnt, ref_cnt); - EXPECT(cnt == ref_cnt); -} diff --git a/cccl_upstream/cudax/examples/stf/word_count_reduce.cu b/cccl_upstream/cudax/examples/stf/word_count_reduce.cu deleted file mode 100644 index 3bc5e726..00000000 --- a/cccl_upstream/cudax/examples/stf/word_count_reduce.cu +++ /dev/null @@ -1,68 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDASTF in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/** - * @file - * @brief Counting words in a text using a launch kernel - */ - -#include - -using namespace cuda::experimental::stf; - -// determines whether the character is alphabetical -__host__ __device__ bool is_alpha(const char c) -{ - return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); -} - -int main() -{ - // Paragraph from 'The Raven' by Edgar Allan Poe - // http://en.wikipedia.org/wiki/The_Raven - const char raw_input[] = - " But the raven, sitting lonely on the placid bust, spoke only,\n" - " That one word, as if his soul in that one word he did outpour.\n" - " Nothing further then he uttered - not a feather then he fluttered -\n" - " Till I scarcely more than muttered `Other friends have flown before -\n" - " On the morrow he will leave me, as my hopes have flown before.'\n" - " Then the bird said, `Nevermore.'\n"; - - context ctx; - - size_t text_len = sizeof(raw_input); - auto ltext = ctx.logical_data(const_cast(&raw_input[0]), {text_len}); - auto lcnt = ctx.logical_data(shape_of>()); - - ctx.parallel_for(box(text_len - 1), ltext.read(), lcnt.reduce(reducer::sum{})) - ->*[] _CCCL_DEVICE(size_t i, auto text, int& s) { - /* When we have the beginning of a new word, increment the counter */ - if (!is_alpha(text(i)) && is_alpha(text(i + 1))) - { - s++; - } - }; - - int cnt = ctx.wait(lcnt); - printf("Got %d words.\n", cnt); - - ctx.finalize(); - - int ref_cnt = 0; - for (size_t i = 0; i < sizeof(raw_input) - 1; i++) - { - if (!is_alpha(raw_input[i]) && is_alpha(raw_input[i + 1])) - { - ref_cnt++; - } - } - - _CCCL_ASSERT(cnt == ref_cnt, "Count mismatch"); -} diff --git a/cccl_upstream/cudax/examples/vector.cuh b/cccl_upstream/cudax/examples/vector.cuh deleted file mode 100644 index 811c7656..00000000 --- a/cccl_upstream/cudax/examples/vector.cuh +++ /dev/null @@ -1,159 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX__CONTAINER_VECTOR -#define _CUDAX__CONTAINER_VECTOR - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include - -#include -#include -#include - -#include -#include - -#include - -namespace cuda::experimental -{ -using ::cuda::std::span; -using ::thrust::device_vector; -using ::thrust::host_vector; - -template -class vector -{ -public: - vector() = default; - explicit vector(size_t __n) - : __h_(__n) - {} - - _Ty& operator[](size_t __i) noexcept - { - __dirty_ = true; - return __h_[__i]; - } - - const _Ty& operator[](size_t __i) const noexcept - { - return __h_[__i]; - } - -private: - void sync_host_to_device([[maybe_unused]] ::cuda::stream_ref __str, __detail::__param_kind __p) const - { - if (__dirty_) - { - if (__p == __detail::__param_kind::_out) - { - // There's no need to copy the data from host to device if the data is - // only going to be written to. We can just allocate the device memory. - __d_.resize(__h_.size()); - } - else - { - // TODO: use a memcpy async here - __d_ = __h_; - } - __dirty_ = false; - } - } - - void sync_device_to_host(::cuda::stream_ref __str, __detail::__param_kind __p) const - { - if (__p != __detail::__param_kind::_in) - { - // TODO: use a memcpy async here - __str.sync(); // wait for the kernel to finish executing - __h_ = __d_; - } - } - - template <__detail::__param_kind _Kind> - class __action //: private __detail::__immovable - { - using __cv_vector = ::cuda::std::__maybe_const<_Kind == __detail::__param_kind::_in, vector>; - - public: - explicit __action(::cuda::stream_ref __str, __cv_vector& __v) - : __str_(__str) - , __v_(__v) - { - __v_.sync_host_to_device(__str_, _Kind); - } - - __action(__action&&) = delete; - - ~__action() - { - try - { - __v_.sync_device_to_host(__str_, _Kind); - } - catch (const ::std::exception& e) - { - static_cast( - ::fprintf(stderr, "Exception occurred during host to device synchronization: %s\n", e.what())); - } - catch (...) - { - static_cast(::fprintf(stderr, "Unknown exception occurred during host to device synchronization\n")); - } - } - - ::cuda::std::span<_Ty> transformed_argument() const - { - return {__v_.__d_.data().get(), __v_.__d_.size()}; - } - - private: - ::cuda::stream_ref __str_; - __cv_vector& __v_; - }; - - [[nodiscard]] friend __action<__detail::__param_kind::_inout> - transform_launch_argument(::cuda::stream_ref __str, vector& __v) - { - return __action<__detail::__param_kind::_inout>{__str, __v}; - } - - [[nodiscard]] friend __action<__detail::__param_kind::_in> - transform_launch_argument(::cuda::stream_ref __str, const vector& __v) - { - return __action<__detail::__param_kind::_in>{__str, __v}; - } - - template <__detail::__param_kind _Kind> - [[nodiscard]] friend __action<_Kind> - transform_launch_argument(::cuda::stream_ref __str, __detail::__box __b) - { - return __action<_Kind>{__str, __b.__val}; - } - - mutable host_vector<_Ty> __h_; - mutable device_vector<_Ty> __d_{}; - mutable bool __dirty_ = true; -}; -} // namespace cuda::experimental - -#endif diff --git a/cccl_upstream/cudax/examples/vector_add.cu b/cccl_upstream/cudax/examples/vector_add.cu deleted file mode 100644 index 6a78f17a..00000000 --- a/cccl_upstream/cudax/examples/vector_add.cu +++ /dev/null @@ -1,127 +0,0 @@ -/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of NVIDIA CORPORATION nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY - * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * Vector addition: C = A + B. - * - * This sample is a very basic sample that implements element by element - * vector addition. It is the same as the sample illustrating Chapter 2 - * of the programming guide with some additions like error checking. - */ - -#include - -// For the CUDA runtime routines (prefixed with "cuda_") -#include - -#include -#include - -#include - -#include "vector.cuh" - -namespace cudax = cuda::experimental; -using cudax::in; -using cudax::out; - -/** - * CUDA Kernel Device code - * - * Computes the vector addition of A and B into C. The 3 vectors have the same - * number of elements numElements. - */ -__global__ void vectorAdd(cudax::span A, cudax::span B, cudax::span C) -{ - int i = static_cast(blockDim.x * blockIdx.x + threadIdx.x); - - if (i < A.size()) - { - C[i] = A[i] + B[i] + 0.0f; - } -} - -/** - * Host main routine - */ -int main() -try -{ - // A CUDA stream on which to execute the vector addition kernel - cudax::stream stream(cuda::devices[0]); - - // Print the vector length to be used, and compute its size - int numElements = 50000; - printf("[Vector addition of %d elements]\n", numElements); - - // Allocate the host vectors - cudax::vector A(numElements); // input - cudax::vector B(numElements); // input - cudax::vector C(numElements); // output - - // Initialize the host input vectors - for (int i = 0; i < numElements; ++i) - { - A[i] = static_cast(rand()) / (float) RAND_MAX; - B[i] = static_cast(rand()) / (float) RAND_MAX; - } - - // Define the kernel launch parameters - constexpr int threadsPerBlock = 256; - auto config = cuda::distribute(numElements); - - // Launch the vectorAdd kernel - printf("CUDA kernel launch with %zu blocks of %d threads\n", cuda::block.count(cuda::grid, config), threadsPerBlock); - cudax::launch(stream, config, vectorAdd, in(A), in(B), out(C)); - - printf("waiting for the stream to finish\n"); - stream.sync(); - - printf("verifying the results\n"); - // Verify that the result vector is correct - for (int i = 0; i < numElements; ++i) - { - if (fabs(A[i] + B[i] - C[i]) > 1e-5) - { - fprintf(stderr, "Result verification failed at element %d!\n", i); - exit(EXIT_FAILURE); - } - } - - printf("Test PASSED\n"); - - printf("Done\n"); - return 0; -} -catch (const std::exception& e) -{ - printf("caught an exception: \"%s\"\n", e.what()); -} -catch (...) -{ - printf("caught an unknown exception\n"); -} diff --git a/cccl_upstream/cudax/include/cuda/experimental/__container/graph_buffer.cuh b/cccl_upstream/cudax/include/cuda/experimental/__container/graph_buffer.cuh deleted file mode 100644 index ad65bff4..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__container/graph_buffer.cuh +++ /dev/null @@ -1,318 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX__CONTAINER_GRAPH_BUFFER_CUH -#define _CUDAX__CONTAINER_GRAPH_BUFFER_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#if _CCCL_CTK_AT_LEAST(12, 2) - -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include - -# include -# include -# include -# include -# include - -# include - -namespace cuda::experimental -{ -//! @rst -//! .. _cudax-container-graph-buffer: -//! -//! Graph buffer -//! ------------ -//! -//! ``graph_buffer`` provides typed device memory allocated as a CUDA graph node. -//! It mirrors the API of ``cuda::buffer`` but takes a ``path_builder&`` instead of -//! a ``stream_ref``. Allocation inserts a ``cuGraphAddMemAllocNode`` into the graph. -//! -//! Memory can be freed in three ways: -//! - ``destroy(path_builder&)`` — inserts a free node into the graph -//! - ``destroy(stream_ref)`` — frees asynchronously on a stream (for memory that outlives the graph) -//! - Destructor — frees on the stored stream if one was set via ``set_stream()`` -//! -//! If the destructor runs with no stream set and the buffer is non-empty, it asserts -//! in debug mode. In release mode the memory leaks. -//! -//! @endrst -//! @tparam _Tp The element type stored in the buffer. Must be trivially copyable. -template -class graph_buffer -{ - static_assert(::cuda::is_trivially_copyable_v<_Tp>, "graph_buffer requires T to be trivially copyable."); - -public: - using value_type = _Tp; - using pointer = _Tp*; - using const_pointer = const _Tp*; - using size_type = ::cuda::std::size_t; - using properties_list = ::cuda::mr::properties_list<::cuda::mr::device_accessible>; - -private: - graph_memory_resource __mr_; - size_type __count_ = 0; - _Tp* __buf_ = nullptr; - ::cudaStream_t __stream_ = ::cuda::__invalid_stream(); - - [[nodiscard]] _CCCL_HOST_API pointer __get_data() const noexcept - { - return __buf_; - } - - //! @brief Causes the buffer to be treated as a span when passed to cudax::launch. - [[nodiscard]] _CCCL_HOST_API friend auto transform_launch_argument(::cuda::stream_ref, graph_buffer& __self) noexcept - -> ::cuda::std::span<_Tp> - { - return {__self.__get_data(), __self.__count_}; - } - - //! @brief Causes the buffer to be treated as a const span when passed to cudax::launch. - [[nodiscard]] _CCCL_HOST_API friend auto - transform_launch_argument(::cuda::stream_ref, const graph_buffer& __self) noexcept -> ::cuda::std::span - { - return {__self.__get_data(), __self.__count_}; - } - -public: - graph_buffer() = delete; - - //! @brief Allocates uninitialized storage for \p __count elements. - _CCCL_HOST_API graph_buffer(path_builder& __pb, graph_memory_resource __mr, size_type __count, ::cuda::no_init_t) - : __mr_(::cuda::std::move(__mr)) - , __count_(__count) - , __buf_(__count_ == 0 ? nullptr : static_cast<_Tp*>(__mr_.allocate(__pb, __count_ * sizeof(_Tp), alignof(_Tp)))) - {} - - //! @brief Allocates storage and fills with \p __value. - _CCCL_HOST_API graph_buffer(path_builder& __pb, graph_memory_resource __mr, size_type __count, const _Tp& __value) - : __mr_(::cuda::std::move(__mr)) - , __count_(__count) - , __buf_(__count_ == 0 ? nullptr : static_cast<_Tp*>(__mr_.allocate(__pb, __count_ * sizeof(_Tp), alignof(_Tp)))) - { - if (__count_ > 0) - { - if constexpr (sizeof(_Tp) == 1) - { - ::cuda::std::uint8_t __byte_val = - static_cast<::cuda::std::uint8_t>(reinterpret_cast(__value)); - ::cuda::experimental::fill_bytes(__pb, ::cuda::std::span<_Tp>(__get_data(), __count_), __byte_val); - } - else - { - // TODO: support non-zero multi-byte values via a kernel node - ::cuda::experimental::fill_bytes( - __pb, ::cuda::std::span<_Tp>(__get_data(), __count_), static_cast<::cuda::std::uint8_t>(0)); - } - } - } - - //! @brief Allocates storage and copies from a contiguous span. - _CCCL_HOST_API graph_buffer(path_builder& __pb, graph_memory_resource __mr, ::cuda::std::span __src) - : __mr_(::cuda::std::move(__mr)) - , __count_(__src.size()) - , __buf_(__count_ == 0 ? nullptr : static_cast<_Tp*>(__mr_.allocate(__pb, __count_ * sizeof(_Tp), alignof(_Tp)))) - { - if (__count_ > 0) - { - ::cuda::experimental::copy_bytes(__pb, __src, ::cuda::std::span<_Tp>{__get_data(), __count_}); - } - } - - //! @brief Allocates storage and copies from an initializer list. - _CCCL_HOST_API graph_buffer(path_builder& __pb, graph_memory_resource __mr, ::cuda::std::initializer_list<_Tp> __ilist) - : graph_buffer(__pb, ::cuda::std::move(__mr), ::cuda::std::span{__ilist.begin(), __ilist.size()}) - {} - - graph_buffer(const graph_buffer&) = delete; - graph_buffer& operator=(const graph_buffer&) = delete; - - //! @brief Move-constructs from another graph_buffer. - _CCCL_HOST_API graph_buffer(graph_buffer&& __other) noexcept - : __mr_(::cuda::std::move(__other.__mr_)) - , __count_(::cuda::std::exchange(__other.__count_, 0)) - , __buf_(::cuda::std::exchange(__other.__buf_, nullptr)) - , __stream_(::cuda::std::exchange(__other.__stream_, ::cuda::__invalid_stream())) - {} - - //! @brief Move-assigns from another graph_buffer. - _CCCL_HOST_API graph_buffer& operator=(graph_buffer&& __other) noexcept - { - if (this != &__other) - { - _CCCL_ASSERT(__buf_ == nullptr || __stream_ != ::cuda::__invalid_stream(), - "graph_buffer move-assigned over non-empty buffer with no stream set"); - if (__buf_ != nullptr && __stream_ != ::cuda::__invalid_stream()) - { - destroy(::cuda::stream_ref{__stream_}); - } - __mr_ = ::cuda::std::move(__other.__mr_); - __count_ = ::cuda::std::exchange(__other.__count_, 0); - __buf_ = ::cuda::std::exchange(__other.__buf_, nullptr); - __stream_ = ::cuda::std::exchange(__other.__stream_, ::cuda::__invalid_stream()); - } - return *this; - } - - //! @brief Destructor. Frees device memory on the stored stream if one was set. - _CCCL_HOST_API ~graph_buffer() - { - if (__buf_ != nullptr) - { - _CCCL_ASSERT(__stream_ != ::cuda::__invalid_stream(), - "graph_buffer destroyed with live memory but no stream set. " - "Call set_stream(), destroy(stream_ref), or destroy(path_builder&) before destruction."); - if (__stream_ != ::cuda::__invalid_stream()) - { - destroy(::cuda::stream_ref{__stream_}); - } - } - } - - //! @brief Set the stream to use for automatic cleanup in the destructor. - _CCCL_HOST_API void set_stream(::cuda::stream_ref __stream) noexcept - { - __stream_ = __stream.get(); - } - - //! @brief Returns the stream set for automatic cleanup. - [[nodiscard]] _CCCL_HOST_API ::cuda::stream_ref stream() const noexcept - { - return ::cuda::stream_ref{__stream_}; - } - - //! @brief Insert a free node into the graph to deallocate the buffer. - _CCCL_HOST_API graph_node_ref destroy(path_builder& __pb) - { - if (__buf_ == nullptr) - { - return graph_node_ref{}; - } - - __mr_.deallocate(__pb, __buf_, __count_ * sizeof(_Tp), alignof(_Tp)); - auto __free_node = __pb.get_dependencies()[0]; - __buf_ = nullptr; - __count_ = 0; - return graph_node_ref{__free_node, __pb.get_native_graph_handle()}; - } - - //! @brief Free the buffer's device memory asynchronously on a stream. - _CCCL_HOST_API void destroy(::cuda::stream_ref __stream) - { - if (__buf_ != nullptr) - { - __mr_.deallocate(__stream, __buf_, __count_ * sizeof(_Tp), alignof(_Tp)); - __buf_ = nullptr; - __count_ = 0; - } - } - - [[nodiscard]] _CCCL_HOST_API pointer data() noexcept - { - return __get_data(); - } - - [[nodiscard]] _CCCL_HOST_API const_pointer data() const noexcept - { - return __get_data(); - } - - [[nodiscard]] _CCCL_HOST_API pointer begin() noexcept - { - return __get_data(); - } - - [[nodiscard]] _CCCL_HOST_API const_pointer begin() const noexcept - { - return __get_data(); - } - - [[nodiscard]] _CCCL_HOST_API pointer end() noexcept - { - return __get_data() + __count_; - } - - [[nodiscard]] _CCCL_HOST_API const_pointer end() const noexcept - { - return __get_data() + __count_; - } - - [[nodiscard]] _CCCL_HOST_API constexpr size_type size() const noexcept - { - return __count_; - } - - [[nodiscard]] _CCCL_HOST_API constexpr size_type size_bytes() const noexcept - { - return __count_ * sizeof(_Tp); - } - - [[nodiscard]] _CCCL_HOST_API constexpr bool empty() const noexcept - { - return __count_ == 0; - } - - [[nodiscard]] _CCCL_HOST_API const graph_memory_resource& memory_resource() const noexcept - { - return __mr_; - } -}; - -//! @brief Create a graph_buffer with uninitialized storage. -template -[[nodiscard]] _CCCL_HOST_API graph_buffer<_Tp> -make_buffer(path_builder& __pb, graph_memory_resource __mr, ::cuda::std::size_t __count, ::cuda::no_init_t) -{ - return graph_buffer<_Tp>{__pb, ::cuda::std::move(__mr), __count, ::cuda::no_init}; -} - -//! @brief Create a graph_buffer filled with a value. -template -[[nodiscard]] _CCCL_HOST_API graph_buffer<_Tp> -make_buffer(path_builder& __pb, graph_memory_resource __mr, ::cuda::std::size_t __count, const _Tp& __value) -{ - return graph_buffer<_Tp>{__pb, ::cuda::std::move(__mr), __count, __value}; -} - -//! @brief Create a graph_buffer from a span of data. -template -[[nodiscard]] _CCCL_HOST_API graph_buffer<_Tp> -make_buffer(path_builder& __pb, graph_memory_resource __mr, ::cuda::std::span __src) -{ - return graph_buffer<_Tp>{__pb, ::cuda::std::move(__mr), __src}; -} -} // namespace cuda::experimental - -# include - -#endif // _CCCL_CTK_AT_LEAST(12, 2) - -#endif // _CUDAX__CONTAINER_GRAPH_BUFFER_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__container/uninitialized_buffer.cuh b/cccl_upstream/cudax/include/cuda/experimental/__container/uninitialized_buffer.cuh deleted file mode 100644 index 91a272f8..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__container/uninitialized_buffer.cuh +++ /dev/null @@ -1,292 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef __CUDAX__CONTAINERS_UNINITIALIZED_BUFFER_H -#define __CUDAX__CONTAINERS_UNINITIALIZED_BUFFER_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -//! @file -//! The \c uninitialized_buffer class provides a typed buffer allocated from a given memory resource. -namespace cuda::experimental -{ -//! @rst -//! .. _cudax-containers-uninitialized-buffer: -//! -//! Uninitialized type-safe memory storage -//! --------------------------------------- -//! -//! ``uninitialized_buffer`` provides a typed buffer allocated from a given :ref:`memory resource -//! `. It handles alignment and release of the allocation. -//! The memory is uninitialized, so that a user needs to ensure elements are properly constructed. -//! -//! In addition to being type-safe, ``uninitialized_buffer`` also takes a set of :ref:`properties -//! ` to ensure that e.g. execution space constraints are checked -//! at compile time. However, we can only forward stateless properties. If a user wants to use a stateful one, then they -//! need to implement :ref:`get_property(const device_buffer&, Property) -//! `. -//! -//! @endrst -//! @tparam _Tp the type to be stored in the buffer -//! @tparam _Properties... The properties the allocated memory satisfies -template -class uninitialized_buffer -{ -private: - static_assert(::cuda::mr::__contains_execution_space_property<_Properties...>, - "The properties of cuda::experimental::uninitialized_buffer must contain at least one execution space " - "property!"); - - using __resource = ::cuda::mr::any_synchronous_resource<_Properties...>; - - __resource __mr_; - size_t __count_ = 0; - void* __buf_ = nullptr; - - template - friend class uninitialized_buffer; - - //! @brief Helper to check whether a different buffer still satisfies all properties of this one - template - static constexpr bool __properties_match = - !::cuda::std::is_same_v<::cuda::std::__make_type_set<_Properties...>, - ::cuda::std::__make_type_set<_OtherProperties...>> - && ::cuda::std::__type_set_contains_v<::cuda::std::__make_type_set<_OtherProperties...>, _Properties...>; - - //! @brief Determines the allocation size given the alignment and size of `T` - [[nodiscard]] _CCCL_HIDE_FROM_ABI static constexpr size_t __get_allocation_size(const size_t __count) noexcept - { - constexpr size_t __alignment = alignof(_Tp); - return (__count * sizeof(_Tp) + (__alignment - 1)) & ~(__alignment - 1); - } - - //! @brief Determines the properly aligned start of the buffer given the alignment and size of `T` - [[nodiscard]] _CCCL_HIDE_FROM_ABI _Tp* __get_data() const noexcept - { - constexpr size_t __alignment = alignof(_Tp); - size_t __space = __get_allocation_size(__count_); - void* __ptr = __buf_; - return ::cuda::std::launder( - static_cast<_Tp*>(::cuda::std::align(__alignment, __count_ * sizeof(_Tp), __ptr, __space))); - } - - //! @brief Causes the buffer to be treated as a span when passed to cudax::launch. - //! @pre The buffer must have the cuda::mr::device_accessible property. - template - [[nodiscard]] _CCCL_HIDE_FROM_ABI friend auto - transform_launch_argument(::cuda::stream_ref, uninitialized_buffer& __self) noexcept - _CCCL_TRAILING_REQUIRES(::cuda::std::span<_Tp>)( - ::cuda::std::same_as<_Tp, _Tp2>&& ::cuda::std::__is_included_in_v<::cuda::mr::device_accessible, _Properties...>) - { - return {__self.__get_data(), __self.size()}; - } - - //! @brief Causes the buffer to be treated as a span when passed to cudax::launch - //! @pre The buffer must have the cuda::mr::device_accessible property. - template - [[nodiscard]] _CCCL_HIDE_FROM_ABI friend auto - transform_launch_argument(::cuda::stream_ref, const uninitialized_buffer& __self) noexcept - _CCCL_TRAILING_REQUIRES(::cuda::std::span)( - ::cuda::std::same_as<_Tp, _Tp2>&& ::cuda::std::__is_included_in_v<::cuda::mr::device_accessible, _Properties...>) - { - return {__self.__get_data(), __self.size()}; - } - -public: - using value_type = _Tp; - using reference = _Tp&; - using const_reference = const _Tp&; - using pointer = _Tp*; - using const_pointer = const _Tp*; - using size_type = size_t; - - //! @brief Constructs an \c uninitialized_buffer and allocates sufficient storage for \p __count elements through - //! \p __mr - //! @param __mr The memory resource to allocate the buffer with. - //! @param __count The desired size of the buffer. - //! @note Depending on the alignment requirements of `T` the size of the underlying allocation might be larger - //! than `count * sizeof(T)`. - //! @note Only allocates memory when \p __count > 0 - _CCCL_HIDE_FROM_ABI uninitialized_buffer(__resource __mr, const size_t __count) - : __mr_(::cuda::std::move(__mr)) - , __count_(__count) - , __buf_(__count_ == 0 ? nullptr : __mr_.allocate_sync(__get_allocation_size(__count_), alignof(_Tp))) - {} - - _CCCL_HIDE_FROM_ABI uninitialized_buffer(const uninitialized_buffer&) = delete; - _CCCL_HIDE_FROM_ABI uninitialized_buffer& operator=(const uninitialized_buffer&) = delete; - - //! @brief Move-constructs a \c uninitialized_buffer from \p __other - //! @param __other Another \c uninitialized_buffer - //! Takes ownership of the allocation in \p __other and resets it - _CCCL_HIDE_FROM_ABI uninitialized_buffer(uninitialized_buffer&& __other) noexcept - : __mr_(::cuda::std::move(__other.__mr_)) - , __count_(::cuda::std::exchange(__other.__count_, 0)) - , __buf_(::cuda::std::exchange(__other.__buf_, nullptr)) - {} - - //! @brief Move-constructs a \c uninitialized_buffer from another \c uninitialized_buffer with matching properties - //! @param __other Another \c uninitialized_buffer - //! Takes ownership of the allocation in \p __other and resets it - _CCCL_TEMPLATE(class... _OtherProperties) - _CCCL_REQUIRES(__properties_match<_OtherProperties...>) - _CCCL_HIDE_FROM_ABI uninitialized_buffer(uninitialized_buffer<_Tp, _OtherProperties...>&& __other) noexcept - : __mr_(::cuda::std::move(__other.__mr_)) - , __count_(::cuda::std::exchange(__other.__count_, 0)) - , __buf_(::cuda::std::exchange(__other.__buf_, nullptr)) - {} - - //! @brief Move-assigns a \c uninitialized_buffer from \p __other - //! @param __other Another \c uninitialized_buffer - //! Deallocates the current allocation and then takes ownership of the allocation in \p __other and resets it - _CCCL_HIDE_FROM_ABI uninitialized_buffer& operator=(uninitialized_buffer&& __other) noexcept - { - if (this == ::cuda::std::addressof(__other)) - { - return *this; - } - - if (__buf_) - { - __mr_.deallocate_sync(__buf_, __get_allocation_size(__count_), alignof(_Tp)); - } - - __mr_ = ::cuda::std::move(__other.__mr_); - __count_ = ::cuda::std::exchange(__other.__count_, 0); - __buf_ = ::cuda::std::exchange(__other.__buf_, nullptr); - return *this; - } - - //! @brief Destroys an \c uninitialized_buffer, deallocates the buffer and destroys the memory resource - //! @warning destroy does not destroy any objects that may or may not reside within the buffer. It is the - //! user's responsibility to ensure that all objects within the buffer have been properly destroyed. - _CCCL_HIDE_FROM_ABI void destroy() - { - if (__buf_) - { - __mr_.deallocate_sync(__buf_, __get_allocation_size(__count_), alignof(_Tp)); - __buf_ = nullptr; - __count_ = 0; - } - auto __tmp_mr = ::cuda::std::move(__mr_); - } - - //! @brief Destroys an \c uninitialized_buffer, deallocates the buffer and destroys the memory resource - //! @warning The destructor does not destroy any objects that may or may not reside within the buffer. It is the - //! user's responsibility to ensure that all objects within the buffer have been properly destroyed. - _CCCL_HIDE_FROM_ABI ~uninitialized_buffer() - { - destroy(); - } - - //! @brief Returns an aligned pointer to the first element in the buffer - [[nodiscard]] _CCCL_HIDE_FROM_ABI pointer begin() noexcept - { - return __get_data(); - } - - //! @overload - [[nodiscard]] _CCCL_HIDE_FROM_ABI const_pointer begin() const noexcept - { - return __get_data(); - } - - //! @brief Returns an aligned pointer to the element following the last element of the buffer. - //! This element acts as a placeholder; attempting to access it results in undefined behavior. - [[nodiscard]] _CCCL_HIDE_FROM_ABI pointer end() noexcept - { - return __get_data() + __count_; - } - - //! @overload - [[nodiscard]] _CCCL_HIDE_FROM_ABI const_pointer end() const noexcept - { - return __get_data() + __count_; - } - - //! @brief Returns an aligned pointer to the first element in the buffer - [[nodiscard]] _CCCL_HIDE_FROM_ABI pointer data() noexcept - { - return __get_data(); - } - - //! @overload - [[nodiscard]] _CCCL_HIDE_FROM_ABI const_pointer data() const noexcept - { - return __get_data(); - } - - //! @brief Returns the size of the allocation - [[nodiscard]] _CCCL_HIDE_FROM_ABI constexpr size_type size() const noexcept - { - return __count_; - } - - //! @brief Returns the size of the buffer in bytes - [[nodiscard]] _CCCL_HIDE_FROM_ABI constexpr size_type size_bytes() const noexcept - { - return __count_ * sizeof(_Tp); - } - - //! @rst - //! Returns a \c const reference to the :ref:`any_resource ` - //! that holds the memory resource used to allocate the buffer - //! @endrst - [[nodiscard]] _CCCL_HIDE_FROM_ABI const __resource& memory_resource() const noexcept - { - return __mr_; - } - - //! @brief Forwards the passed Properties - _CCCL_TEMPLATE(class _Property) - _CCCL_REQUIRES((!property_with_value<_Property>) _CCCL_AND ::cuda::std::__is_included_in_v<_Property, _Properties...>) - _CCCL_HIDE_FROM_ABI friend constexpr void get_property(const uninitialized_buffer&, _Property) noexcept {} - - //! @brief Internal method to grow the allocation to a new size \p __count. - //! @param __count The new size of the allocation. - //! @return An \c uninitialized_buffer that holds the previous allocation - //! @warning This buffer must outlive the returned buffer - _CCCL_HIDE_FROM_ABI uninitialized_buffer __replace_allocation(const size_t __count) - { - // Create a new buffer with a reference to the stored memory resource and swap allocation information - uninitialized_buffer __ret{::cuda::mr::synchronous_resource_ref<_Properties...>{__mr_}, __count}; - ::cuda::std::swap(__count_, __ret.__count_); - ::cuda::std::swap(__buf_, __ret.__buf_); - return __ret; - } -}; - -template -using uninitialized_device_buffer = uninitialized_buffer<_Tp, ::cuda::mr::device_accessible>; -} // namespace cuda::experimental - -#include - -#endif //__CUDAX__CONTAINERS_UNINITIALIZED_BUFFER_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__coop/any_of.cuh b/cccl_upstream/cudax/include/cuda/experimental/__coop/any_of.cuh deleted file mode 100644 index c00fa4c3..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__coop/any_of.cuh +++ /dev/null @@ -1,137 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDA_EXPERIMENTAL___COOP_ANY_OF_CUH -#define _CUDA_EXPERIMENTAL___COOP_ANY_OF_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#if !defined(_CCCL_DOXYGEN_INVOKED) - -// todo: Can we make any_of be implemented as reduce(group, data, cuda::std::logical_or{})? - -namespace cuda::experimental::coop -{ -template -_CCCL_DEVICE_API auto __any_of_impl(...) -{ - static_assert(_Dummy, "cudax::coop::any_of is not supported for the group"); -} - -template -[[nodiscard]] _CCCL_DEVICE_API auto -__any_of_impl(::cuda::std::bool_constant<_Broadcasted>, const this_thread<_Hierarchy>&, bool __thread_data) -{ - if constexpr (_Broadcasted) - { - return __thread_data; - } - else - { - return ::cuda::std::optional{__thread_data}; - } -} - -_CCCL_TEMPLATE(bool _Broadcasted, class _Group) -_CCCL_REQUIRES(is_group<_Group> _CCCL_AND ::cuda::std::is_same_v) -[[nodiscard]] _CCCL_DEVICE_API auto -__any_of_impl(::cuda::std::bool_constant<_Broadcasted>, const _Group& __group, bool __thread_data) noexcept -{ - const auto& __mapping_result = __group.__mapping_result(); - const auto __result = static_cast(::__any_sync(__mapping_result.lane_mask().value(), __thread_data)); - if constexpr (_Broadcasted) - { - return __result; - } - else - { - return (gpu_thread.is_root_rank(__group)) ? ::cuda::std::optional{__result} : ::cuda::std::nullopt; - } -} - -_CCCL_TEMPLATE(class _Group, class _Tp) -_CCCL_REQUIRES(::cuda::std::is_same_v<_Tp, bool>) -[[nodiscard]] _CCCL_DEVICE_API ::cuda::std::optional any_of(const _Group& __group, _Tp __thread_data) -{ - _CCCL_ASSERT(gpu_thread.is_part_of(__group), "Only threads that are part of the group can call cudax::coop::any_of"); - return ::cuda::experimental::coop::__any_of_impl(::cuda::std::false_type{}, __group, __thread_data); -} - -_CCCL_TEMPLATE(class _Group, class _Tp) -_CCCL_REQUIRES(::cuda::std::is_same_v<_Tp, bool>) -[[nodiscard]] _CCCL_DEVICE_API bool any_of(broadcasted_t, const _Group& __group, _Tp __thread_data) -{ - _CCCL_ASSERT(gpu_thread.is_part_of(__group), "Only threads that are part of the group can call cudax::coop::any_of"); - return ::cuda::experimental::coop::__any_of_impl(::cuda::std::true_type{}, __group, __thread_data); -} - -_CCCL_TEMPLATE(class _Group, class _Tp, ::cuda::std::size_t _Np) -_CCCL_REQUIRES(::cuda::std::is_same_v<_Tp, bool>) -[[nodiscard]] _CCCL_DEVICE_API ::cuda::std::optional any_of(const _Group& __group, _Tp (&__thread_data)[_Np]) -{ - _CCCL_ASSERT(gpu_thread.is_part_of(__group), "Only threads that are part of the group can call cudax::coop::any_of"); - return ::cuda::experimental::coop::any_of( - __group, ::cuda::std::reduce(__thread_data, __thread_data + _Np, false, ::cuda::std::logical_or{})); -} - -_CCCL_TEMPLATE(class _Group, class _Tp, ::cuda::std::size_t _Np) -_CCCL_REQUIRES(::cuda::std::is_same_v<_Tp, bool>) -[[nodiscard]] _CCCL_DEVICE_API bool any_of(broadcasted_t, const _Group& __group, _Tp (&__thread_data)[_Np]) -{ - _CCCL_ASSERT(gpu_thread.is_part_of(__group), "Only threads that are part of the group can call cudax::coop::any_of"); - return ::cuda::experimental::coop::any_of( - broadcasted, - __group, - ::cuda::std::reduce(__thread_data, __thread_data + _Np, false, ::cuda::std::logical_or{})); -} - -_CCCL_TEMPLATE(class _Group, class _Tp) -_CCCL_REQUIRES((!::cuda::std::is_same_v<_Tp, bool>) ) -auto any_of(const _Group& __group, _Tp __thread_data) = delete; - -_CCCL_TEMPLATE(class _Group, class _Tp) -_CCCL_REQUIRES((!::cuda::std::is_same_v<_Tp, bool>) ) -auto any_of(broadcasted_t, const _Group& __group, _Tp __thread_data) = delete; - -_CCCL_TEMPLATE(class _Group, class _Tp, ::cuda::std::size_t _Np) -_CCCL_REQUIRES((!::cuda::std::is_same_v<_Tp, bool>) ) -auto any_of(const _Group& __group, _Tp (&__thread_data)[_Np]) = delete; - -_CCCL_TEMPLATE(class _Group, class _Tp, ::cuda::std::size_t _Np) -_CCCL_REQUIRES((!::cuda::std::is_same_v<_Tp, bool>) ) -auto any_of(broadcasted_t, const _Group& __group, _Tp (&__thread_data)[_Np]) = delete; -} // namespace cuda::experimental::coop - -#endif // !_CCCL_DOXYGEN_INVOKED - -#include - -#endif // _CUDA_EXPERIMENTAL___COOP_ANY_OF_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__coop/reduce.cuh b/cccl_upstream/cudax/include/cuda/experimental/__coop/reduce.cuh deleted file mode 100644 index c2939e68..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__coop/reduce.cuh +++ /dev/null @@ -1,414 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDA_EXPERIMENTAL___COOP_REDUCE_CUH -#define _CUDA_EXPERIMENTAL___COOP_REDUCE_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -#if !defined(_CCCL_DOXYGEN_INVOKED) - -// todo(dabayer): We share the temporary storage in shared/global memory for all reduce invocations. This is a temporary -// state before we make it a parameter. - -namespace cuda::experimental::coop -{ -template -_CCCL_DEVICE_API auto __reduce_impl(...) -{ - static_assert(_Dummy, "cudax::coop::reduce is not supported for the group"); -} - -template -[[nodiscard]] _CCCL_DEVICE_API auto __reduce_impl( - ::cuda::std::bool_constant<_Broadcasted>, this_thread<_Hierarchy>, _Tp (&__thread_data)[_Np], _RedFn __red_fn) -{ - const auto __result = ::cub::ThreadReduce(__thread_data, __red_fn); - if constexpr (_Broadcasted) - { - return __result; - } - else - { - return ::cuda::std::optional{__result}; - } -} - -template -[[nodiscard]] _CCCL_DEVICE_API auto __reduce_impl( - ::cuda::std::bool_constant<_Broadcasted>, this_warp<_Hierarchy> __group, _Tp (&__thread_data)[_Np], _RedFn __red_fn) -{ - using _BlockExts = decltype(gpu_thread.extents(block, __group.hierarchy())); - constexpr auto __nwarps_in_block = - ::cuda::ceil_div(_BlockExts::static_extent(0) * _BlockExts::static_extent(1) * _BlockExts::static_extent(2), 32); - - using _WarpReduce = ::cub::WarpReduce<_Tp>; - - union _Scratch - { - typename _WarpReduce::TempStorage __warp_reduce_[__nwarps_in_block]; - }; - __shared__ _Scratch __scratch; - - const auto __warp_rank_in_block = __group.rank(block); - const auto __result = _WarpReduce{__scratch.__warp_reduce_[__warp_rank_in_block]}.Reduce(__thread_data, __red_fn); - if constexpr (_Broadcasted) - { - return ::cuda::device::warp_shuffle_idx(__result, 0).data; - } - else - { - return (gpu_thread.is_root_rank(__group)) ? ::cuda::std::optional{__result} : ::cuda::std::nullopt; - } -} - -template -[[nodiscard]] _CCCL_DEVICE_API auto __reduce_impl( - ::cuda::std::bool_constant<_Broadcasted>, this_block<_Hierarchy> __group, _Tp (&__thread_data)[_Np], _RedFn __red_fn) -{ - using _BlockExts = decltype(gpu_thread.extents(block, __group.hierarchy())); - static_assert(_BlockExts::rank_dynamic() == 0, - "cuda::coop::reduce requires the block level to have all static extents."); - - using _BlockReduce = - ::cub::BlockReduce<_Tp, - static_cast(_BlockExts::static_extent(0)), - ::cub::BLOCK_REDUCE_WARP_REDUCTIONS, - static_cast(_BlockExts::static_extent(1)), - static_cast(_BlockExts::static_extent(2))>; - - union _Scratch - { - typename _BlockReduce::TempStorage __block_reduce_; - _Tp __bcast_; - }; - __shared__ _Scratch __scratch; - - const auto __result = _BlockReduce{__scratch.__block_reduce_}.Reduce(__thread_data, __red_fn); - if constexpr (_Broadcasted) - { - if (gpu_thread.is_root_rank(__group)) - { - __scratch.__bcast_ = __result; - } - __group.sync_aligned(); - return __scratch.__bcast_; - } - else - { - return (gpu_thread.is_root_rank(__group)) ? ::cuda::std::optional{__result} : ::cuda::std::nullopt; - } -} - -template -[[nodiscard]] _CCCL_DEVICE_API auto __reduce_impl( - ::cuda::std::bool_constant<_Broadcasted>, this_cluster<_Hierarchy> __group, _Tp (&__thread_data)[_Np], _RedFn __red_fn) -{ - using _ClusterExts = decltype(block.extents(cluster, __group.hierarchy())); - static_assert(_ClusterExts::rank_dynamic() == 0, - "cuda::coop::reduce requires the cluster level to have all static extents."); - - constexpr auto __nblocks_in_cluster = - _ClusterExts::static_extent(0) * _ClusterExts::static_extent(1) * _ClusterExts::static_extent(2); - if constexpr (__nblocks_in_cluster == 1) - { - return ::cuda::experimental::coop::__reduce_impl( - ::cuda::std::bool_constant<_Broadcasted>{}, this_block{__group.hierarchy()}, __thread_data, __red_fn); - } - else - { - using _BlockExts = decltype(gpu_thread.extents(block, __group.hierarchy())); - static_assert(_BlockExts::rank_dynamic() == 0, - "cuda::coop::reduce requires the block level to have all static extents."); - - using _BlockReduce = - ::cub::BlockReduce<_Tp, - static_cast(_BlockExts::static_extent(0)), - ::cub::BLOCK_REDUCE_WARP_REDUCTIONS, - static_cast(_BlockExts::static_extent(1)), - static_cast(_BlockExts::static_extent(2))>; - - using _RootWarpReduce = ::cub::WarpReduce<_Tp>; - struct _RootScratch - { - _Tp __partials_[__nblocks_in_cluster]; - typename _RootWarpReduce::TempStorage __warp_reduce_; - _Tp __bcast_; - }; - - union _Scratch - { - typename _BlockReduce::TempStorage __block_; - _RootScratch __root_; - }; - __shared__ _Scratch __scratch; - - const auto __partial = _BlockReduce{__scratch.__block_}.Reduce(__thread_data, __red_fn); - _Tp __result{}; - NV_IF_TARGET(NV_PROVIDES_SM_90, ({ - const auto __root_scratch = static_cast<_Scratch*>(::__cluster_map_shared_rank(&__scratch, 0)); - auto& __partials_root = __root_scratch->__root_.__partials_; - - __group.sync_aligned(); - if (gpu_thread.is_root_rank(this_block{__group.hierarchy()})) - { - __partials_root[block.rank(__group)] = __partial; - } - __group.sync_aligned(); - - if (warp.is_root_rank(__group)) - { - this_warp __warp{__group.hierarchy()}; - const auto __value = (gpu_thread.rank(__warp) < __nblocks_in_cluster) - ? __scratch.__root_.__partials_[gpu_thread.rank(__warp)] - : ::cuda::identity_element<_RedFn, _Tp>(); - __result = _RootWarpReduce{__scratch.__root_.__warp_reduce_}.Reduce(__value, __red_fn); - } - - if constexpr (_Broadcasted) - { - if (gpu_thread.is_root_rank(__group)) - { - __scratch.__root_.__bcast_ = __result; - } - __group.sync_aligned(); - __result = __root_scratch->__root_.__bcast_; - - // Wait until all threads are done reading the result. - __group.sync_aligned(); - } - })) - - if constexpr (_Broadcasted) - { - return __result; - } - else - { - return (gpu_thread.is_root_rank(__group)) ? ::cuda::std::optional{__result} : ::cuda::std::nullopt; - } - } -} - -template -_CCCL_DEVICE ::cuda::std::array<_Tp, _Np> __reduce_grid_partials; - -template -[[nodiscard]] _CCCL_DEVICE_API auto __reduce_impl( - ::cuda::std::bool_constant<_Broadcasted>, this_grid<_Hierarchy> __group, _Tp (&__thread_data)[_Np], _RedFn __red_fn) -{ - using _GridExts = decltype(cluster.extents(grid, __group.hierarchy())); - static_assert(_GridExts::rank_dynamic() == 0, - "cuda::coop::reduce requires the grid level to have all static extents."); - - constexpr auto __nclusters_in_grid = - _GridExts::static_extent(0) * _GridExts::static_extent(1) * _GridExts::static_extent(2); - - this_cluster __cluster{__group.hierarchy()}; - const auto __partial = - ::cuda::experimental::coop::__reduce_impl(::cuda::std::false_type{}, __cluster, __thread_data, __red_fn); - - if (gpu_thread.is_root_rank(__cluster)) - { - __reduce_grid_partials<_Tp, __nclusters_in_grid>[cluster.rank(__group)] = __partial.value(); - } - __group.sync_aligned(); - - ::cuda::std::optional<_Tp> __result; - if (block.is_root_rank(__group)) - { - this_block __block{__group.hierarchy()}; - - constexpr auto __npartials_per_thread = ::cuda::ceil_div(__nclusters_in_grid, gpu_thread.static_count(__block)); - _Tp __thread_partials[__npartials_per_thread]; - const auto __offset = gpu_thread.rank(__block) * __npartials_per_thread; - - // todo(dabayer): This is not the most efficient way to load values, it doesn't take into account element size and - // reads N consecutive elements by 1 thread. - for (unsigned __i = 0; __i < __npartials_per_thread; ++__i) - { - __thread_partials[__i] = - (__offset + __i < __nclusters_in_grid) - ? __reduce_grid_partials<_Tp, __nclusters_in_grid>[__offset + __i] - : ::cuda::identity_element<_RedFn, _Tp>(); - } - __result = - ::cuda::experimental::coop::__reduce_impl(::cuda::std::false_type{}, __block, __thread_partials, __red_fn); - } - - if constexpr (_Broadcasted) - { - if (gpu_thread.is_root_rank(__group)) - { - __reduce_grid_partials<_Tp, __nclusters_in_grid>[0] = *__result; - } - __group.sync_aligned(); - const auto __result2 = __reduce_grid_partials<_Tp, __nclusters_in_grid>[0]; - - // Wait until all threads are done reading the result. - __group.sync_aligned(); - return __result2; - } - else - { - return __result; - } -} - -_CCCL_TEMPLATE(bool _Broadcasted, class _Group, class _Tp, ::cuda::std::size_t _Np, class _RedFn) -_CCCL_REQUIRES(::cuda::std::is_same_v - _CCCL_AND ::cuda::std::is_same_v) -[[nodiscard]] _CCCL_DEVICE_API auto -__reduce_impl(::cuda::std::bool_constant<_Broadcasted>, _Group __group, _Tp (&__thread_data)[_Np], _RedFn __red_fn) -{ - using _MappingResult = typename _Group::__mapping_result_type; - const auto& __mapping_result = __group.__mapping_result(); - - const auto __lane_mask = __mapping_result.lane_mask(); - const auto __lane = ::cuda::ptx::get_sreg_laneid(); - auto __result = ::cub::ThreadReduce(__thread_data, __red_fn); - - _CCCL_PRAGMA_UNROLL_FULL() - for (unsigned __stride = 1; __stride < ::cuda::next_power_of_two(__mapping_result.unit_count()); __stride *= 2) - { - const auto __other = ::cuda::experimental::coop::shuffle_down(__group, __result, __stride); - if (__other.has_value()) - { - __result = __red_fn(__result, *__other); - } - } - - if constexpr (_Broadcasted) - { - return ::cuda::device::warp_shuffle_idx(__result, ::cuda::std::countr_zero(__lane_mask.value()), __lane_mask.value()) - .data; - } - else - { - return (__mapping_result.unit_rank() == 0) ? ::cuda::std::optional{__result} : ::cuda::std::nullopt; - } -} - -_CCCL_TEMPLATE(bool _Broadcasted, class _Group, class _Tp, ::cuda::std::size_t _Np, class _RedFn) -_CCCL_REQUIRES(::cuda::std::is_same_v - _CCCL_AND ::cuda::std::is_same_v) -[[nodiscard]] _CCCL_DEVICE_API auto -__reduce_impl(::cuda::std::bool_constant<_Broadcasted>, _Group __group, _Tp (&__thread_data)[_Np], _RedFn __red_fn) -{ - constexpr auto __nwarps_in_group = warp.static_count(__group); - static_assert(__nwarps_in_group != ::cuda::std::dynamic_extent, - "cuda::coop::reduce requires the group to have statically known size"); - - using _WarpReduce = ::cub::WarpReduce<_Tp>; - struct _AdditionalScratch - { - _Tp __partials_[__nwarps_in_group]; - _Tp __bcast_; - }; - - union _Scratch - { - typename _WarpReduce::TempStorage __warp_reduce_[__nwarps_in_group]; - _AdditionalScratch __additional_; - }; - __shared__ _Scratch __scratch; - - const auto __partial = _WarpReduce{__scratch.__warp_reduce_[warp.rank(__group)]}.Reduce(__thread_data, __red_fn); - __group.sync_aligned(); - - this_warp __warp{__group.hierarchy()}; - if (gpu_thread.is_root_rank(__warp)) - { - __scratch.__additional_.__partials_[warp.rank(__group)] = __partial; - } - __group.sync_aligned(); - - _Tp __result; - if (warp.is_root_rank(__group)) - { - const auto __value = (gpu_thread.rank(__warp) < __nwarps_in_group) - ? __scratch.__additional_.__partials_[gpu_thread.rank(__warp)] - : ::cuda::identity_element<_RedFn, _Tp>(); - __result = _WarpReduce{__scratch.__warp_reduce_[0]}.Reduce(__value, __red_fn); - } - - if constexpr (_Broadcasted) - { - if (gpu_thread.is_root_rank(__group)) - { - __scratch.__additional_.__bcast_ = __result; - } - __group.sync_aligned(); - return __scratch.__additional_.__bcast_; - } - else - { - return (gpu_thread.is_root_rank(__group)) ? ::cuda::std::optional{__result} : ::cuda::std::nullopt; - } -} - -template -[[nodiscard]] _CCCL_DEVICE_API ::cuda::std::optional<_Tp> -reduce(_Group __group, _Tp (&__thread_data)[_Np], _RedFn __red_fn) -{ - static_assert(gpu_thread.static_count(__group) != ::cuda::std::dynamic_extent, - "cuda::coop::reduce requires the group to have statically known size"); - - _CCCL_ASSERT(gpu_thread.is_part_of(__group), "Only threads that are part of the group can call cudax::coop::reduce"); - - return ::cuda::experimental::coop::__reduce_impl(::cuda::std::false_type{}, __group, __thread_data, __red_fn); -} - -template -[[nodiscard]] _CCCL_DEVICE_API _Tp reduce(broadcasted_t, _Group __group, _Tp (&__thread_data)[_Np], _RedFn __red_fn) -{ - static_assert(gpu_thread.static_count(__group) != ::cuda::std::dynamic_extent, - "cuda::coop::reduce requires the group to have statically known size"); - - _CCCL_ASSERT(gpu_thread.is_part_of(__group), "Only threads that are part of the group can call cudax::coop::reduce"); - - return ::cuda::experimental::coop::__reduce_impl(::cuda::std::true_type{}, __group, __thread_data, __red_fn); -} -} // namespace cuda::experimental::coop - -#endif // !_CCCL_DOXYGEN_INVOKED - -#include - -#endif // _CUDA_EXPERIMENTAL___COOP_REDUCE_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__coop/shuffle.cuh b/cccl_upstream/cudax/include/cuda/experimental/__coop/shuffle.cuh deleted file mode 100644 index 8803ea08..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__coop/shuffle.cuh +++ /dev/null @@ -1,85 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDA_EXPERIMENTAL___COOP_SHUFFLE_CUH -#define _CUDA_EXPERIMENTAL___COOP_SHUFFLE_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include - -#include - -#include - -#if !defined(_CCCL_DOXYGEN_INVOKED) - -namespace cuda::experimental::coop -{ -template -[[nodiscard]] _CCCL_DEVICE_API auto __shuffle_impl(...) -{ - static_assert(_Dummy, "cudax::coop::shuffle is not implemented for this group"); -} - -_CCCL_TEMPLATE(class _Group, class _Tp) -_CCCL_REQUIRES(is_group<_Group> _CCCL_AND ::cuda::std::is_same_v - _CCCL_AND ::cuda::std::is_same_v) -[[nodiscard]] _CCCL_DEVICE_API _Tp __shuffle_impl(const _Group& __group, _Tp __value, unsigned __src_unit_rank) noexcept -{ - using _MappingResult = typename _Group::__mapping_result_type; - const auto& __mapping_result = __group.__mapping_result(); - - _CCCL_ASSERT(__src_unit_rank < __mapping_result.unit_count(), - "invalid __src_unit_rank - must be less than the number of units within the group"); - - const auto __lane_mask = __mapping_result.lane_mask(); - const auto __lane_offset = static_cast(__src_unit_rank) - static_cast(__mapping_result.unit_rank()); - - unsigned __src_lane{}; - if constexpr (_MappingResult::is_always_contiguous()) - { - const auto __lane = ::cuda::ptx::get_sreg_laneid(); - __src_lane = static_cast(__lane + __lane_offset); - } - else - { - __src_lane = ::__fns(__lane_mask.value(), 0, static_cast(__src_unit_rank) + 1); - } - return ::cuda::device::warp_shuffle_idx(__value, static_cast(__src_lane), __lane_mask.value()); -} - -//! @brief Shuffles values among units within a group. -//! @param[in] __group The group. -//! @param[in] __value This thread's value to be shuffled. -//! @param[in] __src_unit_rank The rank of the unit whose value should be taken by this unit. -//! @return The value passed to the function by the equivalent thread from the source rank unit. -template -[[nodiscard]] _CCCL_DEVICE_API _Tp shuffle(const _Group& __group, _Tp __value, unsigned __src_unit_rank) noexcept -{ - return ::cuda::experimental::coop::__shuffle_impl(__group, __value, __src_unit_rank); -} -} // namespace cuda::experimental::coop - -#endif // !_CCCL_DOXYGEN_INVOKED - -#include - -#endif // _CUDA_EXPERIMENTAL___COOP_SHUFFLE_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__coop/shuffle_down.cuh b/cccl_upstream/cudax/include/cuda/experimental/__coop/shuffle_down.cuh deleted file mode 100644 index 067b74b1..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__coop/shuffle_down.cuh +++ /dev/null @@ -1,89 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDA_EXPERIMENTAL___COOP_SHUFFLE_DOWN_CUH -#define _CUDA_EXPERIMENTAL___COOP_SHUFFLE_DOWN_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include - -#include - -#include - -#if !defined(_CCCL_DOXYGEN_INVOKED) - -namespace cuda::experimental::coop -{ -template -[[nodiscard]] _CCCL_DEVICE_API auto __shuffle_down_impl(...) -{ - static_assert(_Dummy, "cudax::coop::shuffle_down is not implemented for this group"); -} - -_CCCL_TEMPLATE(class _Group, class _Tp) -_CCCL_REQUIRES(is_group<_Group> _CCCL_AND ::cuda::std::is_same_v - _CCCL_AND ::cuda::std::is_same_v) -[[nodiscard]] _CCCL_DEVICE_API ::cuda::std::optional<_Tp> -__shuffle_down_impl(const _Group& __group, const _Tp& __value, unsigned __offset) noexcept -{ - using _MappingResult = typename _Group::__mapping_result_type; - const auto& __mapping_result = __group.__mapping_result(); - - const auto __lane_mask = __mapping_result.lane_mask(); - const auto __offset_is_valid = (__offset < __mapping_result.unit_count() - __mapping_result.unit_rank()); - - if constexpr (_MappingResult::is_always_contiguous()) - { - const auto __real_offset = (__offset_is_valid) ? __offset : 0u; - const auto __result = - ::cuda::device::warp_shuffle_down(__value, static_cast(__real_offset), __lane_mask.value()); - return (__offset_is_valid) ? ::cuda::std::optional{__result.data} : ::cuda::std::nullopt; - } - else - { - const auto __lane = ::cuda::ptx::get_sreg_laneid(); - const auto __src_lane = - (__offset_is_valid) ? ::__fns(__lane_mask.value(), __lane, static_cast(__offset + 1)) : __lane; - const auto __result = ::cuda::device::warp_shuffle_idx(__value, static_cast(__src_lane), __lane_mask.value()); - return (__offset_is_valid) ? ::cuda::std::optional{__result.data} : ::cuda::std::nullopt; - } -} - -//! @brief Gets the values from a unit with a greater rank by the specified offset. -//! @param[in] __group The group. -//! @param[in] __value This thread's value. -//! @param[in] __offset The offset of the source unit rank from this unit's rank. -//! @return The source's value or empty optional if no such rank exists. -template -[[nodiscard]] _CCCL_DEVICE_API ::cuda::std::optional<_Tp> -shuffle_down(const _Group& __group, const _Tp& __value, unsigned __offset) noexcept -{ - return ::cuda::experimental::coop::__shuffle_down_impl(__group, __value, __offset); -} -} // namespace cuda::experimental::coop - -#endif // !_CCCL_DOXYGEN_INVOKED - -#include - -#endif // _CUDA_EXPERIMENTAL___COOP_SHUFFLE_DOWN_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__coop/shuffle_up.cuh b/cccl_upstream/cudax/include/cuda/experimental/__coop/shuffle_up.cuh deleted file mode 100644 index c5e6a73a..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__coop/shuffle_up.cuh +++ /dev/null @@ -1,91 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDA_EXPERIMENTAL___COOP_SHUFFLE_UP_CUH -#define _CUDA_EXPERIMENTAL___COOP_SHUFFLE_UP_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include - -#include - -#include - -#if !defined(_CCCL_DOXYGEN_INVOKED) - -namespace cuda::experimental::coop -{ -template -[[nodiscard]] _CCCL_DEVICE_API auto __shuffle_up_impl(...) -{ - static_assert(_Dummy, "cudax::coop::shuffle_up is not implemented for this group"); -} - -_CCCL_TEMPLATE(class _Group, class _Tp) -_CCCL_REQUIRES(is_group<_Group> _CCCL_AND ::cuda::std::is_same_v - _CCCL_AND ::cuda::std::is_same_v) -[[nodiscard]] _CCCL_DEVICE_API ::cuda::std::optional<_Tp> -__shuffle_up_impl(const _Group& __group, const _Tp& __value, unsigned __offset) noexcept -{ - using _MappingResult = typename _Group::__mapping_result_type; - const auto& __mapping_result = __group.__mapping_result(); - - const auto __lane_mask = __mapping_result.lane_mask(); - const auto [__src_rank, __underflow] = ::cuda::sub_overflow(__mapping_result.unit_rank(), __offset); - const auto __offset_is_valid = !__underflow; - - if constexpr (_MappingResult::is_always_contiguous()) - { - const auto __real_offset = (__offset_is_valid) ? __offset : 0u; - const auto __result = - ::cuda::device::warp_shuffle_up(__value, static_cast(__real_offset), __lane_mask.value()); - return (__offset_is_valid) ? ::cuda::std::optional{__result.data} : ::cuda::std::nullopt; - } - else - { - const auto __lane = ::cuda::ptx::get_sreg_laneid(); - const auto __src_lane = - (__offset_is_valid) ? ::__fns(__lane_mask.value(), 0, static_cast(__src_rank + 1)) : __lane; - const auto __result = ::cuda::device::warp_shuffle_idx(__value, static_cast(__src_lane), __lane_mask.value()); - return (__offset_is_valid) ? ::cuda::std::optional{__result.data} : ::cuda::std::nullopt; - } -} - -//! @brief Gets the values from a unit with a lower rank by the specified offset. -//! @param[in] __group The group. -//! @param[in] __value This thread's value. -//! @param[in] __offset The offset of the source unit rank from this unit's rank. -//! @return The source's value or empty optional if no such rank exists. -template -[[nodiscard]] _CCCL_DEVICE_API ::cuda::std::optional<_Tp> -shuffle_up(const _Group& __group, const _Tp& __value, unsigned __offset) noexcept -{ - return ::cuda::experimental::coop::__shuffle_up_impl(__group, __value, __offset); -} -} // namespace cuda::experimental::coop - -#endif // !_CCCL_DOXYGEN_INVOKED - -#include - -#endif // _CUDA_EXPERIMENTAL___COOP_SHUFFLE_UP_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy/copy_contiguous.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy/copy_contiguous.cuh deleted file mode 100644 index 5c5280eb..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy/copy_contiguous.cuh +++ /dev/null @@ -1,236 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX__COPY_CONTIGUOUS_H -#define _CUDAX__COPY_CONTIGUOUS_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -namespace cuda::experimental -{ -//! @brief Tiled copy kernel for contiguous innermost dimension. -//! -//! Uses a 2D grid: blockIdx.x = tile along inner dimension, blockIdx.y = outer index. -//! Threads within a block stride over the tile, reading from the source and writing to the -//! destination via accessors. The coordinate iterator maps linear indices to multi-dimensional -//! coordinates, which are then used with per-tensor strides for the actual memory access. -//! -//! @param[in] __config Kernel launch configuration -//! @param[in] __src_ptr Pointer to source data -//! @param[in] __src_strides Per-dimension strides for the source tensor -//! @param[in] __src_accessor Accessor for reading source elements -//! @param[out] __dst_ptr Pointer to destination data -//! @param[in] __dst_strides Per-dimension strides for the destination tensor -//! @param[in] __dst_accessor Accessor for writing destination elements -//! @param[in] __coord_iter Coordinate iterator for multi-dimensional index mapping -//! @param[in] __inner_size Extent of the contiguous innermost dimension -template -__global__ void __copy_contiguous_kernel( - _CCCL_GRID_CONSTANT const _Config __config, - _CCCL_GRID_CONSTANT const _TpSrc* const _CCCL_RESTRICT __src_ptr, - _CCCL_GRID_CONSTANT const ::cuda::std::array<_StrideTIn, _Rank> __src_strides, - _CCCL_GRID_CONSTANT const _SrcAccessor __src_accessor, - _CCCL_GRID_CONSTANT _TpDst* const _CCCL_RESTRICT __dst_ptr, - _CCCL_GRID_CONSTANT const ::cuda::std::array<_StrideTOut, _Rank> __dst_strides, - _CCCL_GRID_CONSTANT const _DstAccessor __dst_accessor, - _CCCL_GRID_CONSTANT const __tensor_coord_iterator<_ExtentT, _Rank> __coord_iter, - _CCCL_GRID_CONSTANT const _ExtentT __inner_size) -{ - using __partial_tensor_src = __partial_tensor; - using __partial_tensor_dst = __partial_tensor<_TpDst, _StrideTOut, _Rank, _DstAccessor>; - const auto __thread_id = ::cuda::gpu_thread.rank_as<_ExtentT>(::cuda::block, __config); - const auto __block_idx = ::cuda::block.index_as<_ExtentT>(::cuda::grid); - constexpr auto __block_size = ::cuda::gpu_thread.count_as(::cuda::block, __config); - const __partial_tensor_src __src{__src_ptr, __src_strides, __src_accessor}; - const __partial_tensor_dst __dst{__dst_ptr, __dst_strides, __dst_accessor}; - - const auto __tile_offset = __block_idx.x * _TileSize; - const auto __outer_idx = __block_idx.y; - const auto __remaining = __inner_size - __tile_offset; - const auto __base_idx = __outer_idx * __inner_size + __tile_offset + __thread_id; - - if (__remaining >= _TileSize) - { - _CCCL_PRAGMA_UNROLL_FULL() - for (int __i = 0; __i < _TileSize; __i += __block_size) - { - const auto __coord = __coord_iter(__base_idx + __i); - __dst(__coord) = __src(__coord); - } - } - else - { - _CCCL_PRAGMA_UNROLL_FULL() - for (int __i = 0; __i < _TileSize; __i += __block_size) - { - if (__thread_id + __i < __remaining) - { - const auto __coord = __coord_iter(__base_idx + __i); - __dst(__coord) = __src(__coord); - } - } - } -} - -//! @brief Query the minimum bytes-in-flight target for the current GPU architecture. -//! -//! Delegates to CUB's architecture-specific tuning. -//! @return Bytes-in-flight target (e.g. 12KB for V100, 16KB for A100, 48KB for H200, 64KB for B200) -[[nodiscard]] _CCCL_HOST_API inline int __bytes_in_flight() noexcept -{ - const auto __dev_id = ::cuda::__driver::__cudevice_to_ordinal(::cuda::__driver::__ctxGetDevice()); - const auto __dev = ::cuda::devices[__dev_id]; - const auto __cc = ::cuda::device_attributes::compute_capability(__dev); - return CUB_NS_QUALIFIER::detail::transform::cc_to_min_bytes_in_flight(__cc); -} - -// Compute the number of elements each thread copies for a given vector width. -[[nodiscard]] _CCCL_HOST_API inline int __elem_per_thread(int __access_bytes, int __bytes_in_flight) noexcept -{ - constexpr auto __threads_per_sm = 2048; - return ::cuda::std::max(__bytes_in_flight / (__access_bytes * __threads_per_sm), 1); -} - -// Dispatch a callable with a compile-time tile size derived from a runtime value. -template -_CCCL_HOST_API void __dispatch_tile_size(int __tile_size, _Op __op) noexcept -{ - if (__tile_size >= 2048) - { - __op(::cuda::std::integral_constant{}); - } - else if (__tile_size >= 1024) - { - __op(::cuda::std::integral_constant{}); - } - else if (__tile_size >= 512) - { - __op(::cuda::std::integral_constant{}); - } - else - { - __op(::cuda::std::integral_constant{}); - } -} - -//! @brief Launch the tiled copy kernel for contiguous innermost dimension. -//! -//! Computes tile size from the architecture-specific bytes-in-flight target, then dispatches -//! the @ref __copy_contiguous_kernel with a compile-time tile size. -//! -//! @param[in] __src Source raw tensor descriptor -//! @param[in] __dst Destination raw tensor descriptor -//! @param[in] __stream CUDA stream for asynchronous execution -//! @param[in] __src_accessor Accessor for reading source elements -//! @param[in] __dst_accessor Accessor for writing destination elements -template , - typename _DstAccessor = ::cuda::std::default_accessor<_TpOut>> -_CCCL_HOST_API void __launch_copy_contiguous_kernel( - const __raw_tensor<_ExtentT, _StrideTIn, _TpIn, _Rank>& __src, - const __raw_tensor<_ExtentT, _StrideTOut, _TpOut, _Rank>& __dst, - ::cuda::stream_ref __stream, - const _SrcAccessor& __src_accessor = {}, - const _DstAccessor& __dst_accessor = {}) -{ - constexpr int __block_size = 256; - const auto __bytes_in_flight = ::cuda::experimental::__bytes_in_flight(); - const auto __elems_per_thread = - ::cuda::experimental::__elem_per_thread(static_cast(sizeof(_TpIn)), __bytes_in_flight); - const auto __tile_size_rt = __block_size * __elems_per_thread; - - ::cuda::experimental::__dispatch_tile_size(__tile_size_rt, [&](auto __tile_constant) { - constexpr int __tile_size = decltype(__tile_constant)::value; - const auto __inner_size = __src.__extents[0]; - const auto __outer_size = ::cuda::experimental::__total_size(__src) / __inner_size; - const auto __num_inner_tiles = ::cuda::ceil_div(__inner_size, __tile_size); - constexpr auto __arch_limits = ::cuda::__common_arch_traits(::cuda::arch_id::sm_90); - _CCCL_ASSERT(__num_inner_tiles <= _ExtentT(__arch_limits.max_grid_dim_x), - "grid x-dimension exceeds the maximum grid size"); - _CCCL_ASSERT(__outer_size <= _ExtentT(__arch_limits.max_grid_dim_y), - "grid y-dimension exceeds the maximum grid size"); - const auto __grid_dims = ::dim3(static_cast(__num_inner_tiles), static_cast(__outer_size)); - const auto __config = ::cuda::make_config(::cuda::block_dims<__block_size>(), ::cuda::grid_dims(__grid_dims)); - - const __tensor_coord_iterator<_ExtentT, _Rank> __coord_iter{__src.__extents}; - const auto __kernel = ::cuda::experimental::__copy_contiguous_kernel< - decltype(__config), - __tile_size, - _TpIn, - _TpOut, - _SrcAccessor, - _DstAccessor, - _ExtentT, - _StrideTIn, - _StrideTOut, - _Rank>; - - ::cuda::launch( - __stream, - __config, - __kernel, - __src.__data, - __src.__strides, - __src_accessor, - __dst.__data, - __dst.__strides, - __dst_accessor, - __coord_iter, - __inner_size); - }); -} -} // namespace cuda::experimental - -#include - -#endif // _CUDAX__COPY_CONTIGUOUS_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy/copy_optimized.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy/copy_optimized.cuh deleted file mode 100644 index 41846003..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy/copy_optimized.cuh +++ /dev/null @@ -1,149 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX__COPY_OPTIMIZED_H -#define _CUDAX__COPY_OPTIMIZED_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -namespace cuda::experimental -{ -//! @brief Element-wise copy kernel for strided tensor data. -//! -//! Each thread copies one element at a time using a grid-stride loop, mapping linear indices to -//! multi-dimensional coordinates via @ref __tensor_coord_iterator. -//! -//! @param[in] __config Kernel launch configuration -//! @param[in] __src_ptr Pointer to source data -//! @param[in] __src_strides Per-dimension strides for the source tensor -//! @param[in] __src_accessor Accessor for reading source elements -//! @param[out] __dst_ptr Pointer to destination data -//! @param[in] __dst_strides Per-dimension strides for the destination tensor -//! @param[in] __dst_accessor Accessor for writing destination elements -//! @param[in] __coord_iter Coordinate iterator for multi-dimensional index mapping -//! @param[in] __tensor_size Total number of elements to copy -template -__global__ void __copy_optimized_kernel( - _CCCL_GRID_CONSTANT const _Config __config, - _CCCL_GRID_CONSTANT const _TpSrc* const _CCCL_RESTRICT __src_ptr, - _CCCL_GRID_CONSTANT const ::cuda::std::array<_StrideTIn, _Rank> __src_strides, - _CCCL_GRID_CONSTANT const _SrcAccessor __src_accessor, - _CCCL_GRID_CONSTANT _TpDst* const _CCCL_RESTRICT __dst_ptr, - _CCCL_GRID_CONSTANT const ::cuda::std::array<_StrideTOut, _Rank> __dst_strides, - _CCCL_GRID_CONSTANT const _DstAccessor __dst_accessor, - _CCCL_GRID_CONSTANT const __tensor_coord_iterator<_ExtentT, _Rank> __coord_iter, - _CCCL_GRID_CONSTANT const _ExtentT __tensor_size) -{ - using __partial_tensor_src = __partial_tensor; - using __partial_tensor_dst = __partial_tensor<_TpDst, _StrideTOut, _Rank, _DstAccessor>; - const auto __idx = ::cuda::gpu_thread.rank_as<_ExtentT>(::cuda::grid, __config); - const auto __stride = ::cuda::gpu_thread.count_as<_ExtentT>(::cuda::grid, __config); - const __partial_tensor_src __src{__src_ptr, __src_strides, __src_accessor}; - const __partial_tensor_dst __dst{__dst_ptr, __dst_strides, __dst_accessor}; - - for (auto __i = __idx; __i < __tensor_size; __i += __stride) - { - const auto __coord = __coord_iter(__i); - __dst(__coord) = __src(__coord); - if constexpr (sizeof(_ExtentT) <= 4) - { - return; - } - } -} - -//! @brief Launch a naive element-wise copy kernel for strided tensor data. -//! -//! Each thread copies one element at a time using a grid-stride loop. Coordinates are -//! computed from linear indices via @ref __tensor_coord_iterator. -//! -//! @param[in] __src Source raw tensor descriptor -//! @param[out] __dst Destination raw tensor descriptor -//! @param[in] __tensor_size Total number of elements to copy -//! @param[in] __stream CUDA stream for asynchronous execution -//! @param[in] __src_accessor Accessor for reading source elements -//! @param[in] __dst_accessor Accessor for writing destination elements -template , - typename _DstAccessor = ::cuda::std::default_accessor<_TpOut>> -_CCCL_HOST_API void __copy_optimized( - const __raw_tensor<_ExtentT, _StrideTIn, _TpIn, _Rank>& __src, - const __raw_tensor<_ExtentT, _StrideTOut, _TpOut, _Rank>& __dst, - _ExtentT __tensor_size, - ::cuda::stream_ref __stream, - const _SrcAccessor& __src_accessor = {}, - const _DstAccessor& __dst_accessor = {}) noexcept -{ - constexpr int __block_size = 256; - const __tensor_coord_iterator<_ExtentT, _Rank> __coord_iter(__src.__extents); - const auto __grid_size = ::cuda::ceil_div(__tensor_size, _ExtentT{__block_size}); - const auto __config = ::cuda::make_config(::cuda::block_dims<__block_size>(), ::cuda::grid_dims(__grid_size)); - const auto& __kernel = ::cuda::experimental::__copy_optimized_kernel< - decltype(__config), - _TpIn, - _TpOut, - _SrcAccessor, - _DstAccessor, - _ExtentT, - _StrideTIn, - _StrideTOut, - _Rank>; - - ::cuda::launch( - __stream, - __config, - __kernel, - __src.__data, - __src.__strides, - __src_accessor, - __dst.__data, - __dst.__strides, - __dst_accessor, - __coord_iter, - __tensor_size); -} -} // namespace cuda::experimental - -#include - -#endif // _CUDAX__COPY_OPTIMIZED_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy/copy_shared_memory.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy/copy_shared_memory.cuh deleted file mode 100644 index 7649bef7..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy/copy_shared_memory.cuh +++ /dev/null @@ -1,442 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX__COPY_COPY_SHARED_MEMORY_H -#define _CUDAX__COPY_COPY_SHARED_MEMORY_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -//! Shared-memory tiled transpose for arbitrary-rank tensor copies. -//! -//! The overall idea is to decompose the tensors into tiles that can fit in shared memory. -//! Each tile is assigned to a thread block. A tile can entirely represent a dimension or split the respective extent. -//! The algorithm creates tiles over dimensions that provide coalesced accesses in the source and destination tensors. -//! -//! (1) Grid decomposition -//! The tensor is partitioned into tiles whose per-dimension sizes are capped by warp size and shared-memory capacity. -//! The total number of tiles (product of ceil(extent[d] / tile_size[d]) over all dimensions) becomes the 1-D grid size. -//! -//! (2) Block processing -//! Each block handles one tile in two phases: -//! 1. *Load*: threads cooperatively read source elements into shared memory. -//! This requires additional logic to "transpose" the source tensor into a row-major order. -//! The mapping is determined by using the source-tile permutation obtained by sorting by |src stride|. -//! 2. *Store*: after a barrier, threads read shared memory in destination-coalesced order by using the -//! destination-tile permutation obtained by sorting by |dst stride|. -//! -//! Boundary tiles that extend past the tensor extents fall back to a direct element-wise copy without shared memory. - -namespace cuda::experimental -{ -//! @brief Compute the shared-memory offset for the XOR swizzle. -//! -//! @param[in] __offset The offset in the shared-memory tile. -//! @return The offset in the shared-memory tile with the XOR swizzle applied. -template -[[nodiscard]] _CCCL_DEVICE_API __tile_extent_t __smem_offset(__tile_extent_t __offset) noexcept -{ - if constexpr (_UseXorSwizzle) - { - static_assert(__max_tile_size == 32, "XOR shared-memory swizzle assumes 32 banks and 32-element tile modes"); - constexpr __tile_extent_t __swizzle_tile_size = __max_tile_size * __max_tile_size; - const auto __outer = __offset / __swizzle_tile_size; - const auto __offset_tile_rounded = __outer * __swizzle_tile_size; - const auto __inner = __offset - __offset_tile_rounded; - const auto __row = __inner / __max_tile_size; - const auto __row_tile_rounded = __row * __max_tile_size; - const auto __col = __inner - __row_tile_rounded; - return __offset_tile_rounded + __row_tile_rounded + (__col ^ __row); - } - return __offset; -} - -//! @brief Shared-memory tiled transpose kernel for arbitrary-rank tensors. -//! -//! Each block processes one tile. Threads cooperatively iterate over tile elements with a stride loop. Full (interior) -//! tiles use a two-phase shared-memory transpose: load source data into shared memory using source-coalesced ordering, -//! then store from shared memory to destination using destination-coalesced ordering. Partial (boundary) tiles copy -//! elements directly without shared memory. -//! -//! @param[in] __config Kernel launch configuration -//! @param[in] __src_ptr Pointer to source data -//! @param[in] __src_accessor Accessor for reading source elements -//! @param[out] __dst_ptr Pointer to destination data -//! @param[in] __dst_accessor Accessor for writing destination elements -//! @param[in] __grid_iter Coordinate iterator for grid tile decomposition -//! @param[in] __grid_tile_src_strides Per-dimension source strides scaled by tile sizes -//! @param[in] __grid_tile_dst_strides Per-dimension destination strides scaled by tile sizes -//! @param[in] __tile_perm_iter Coordinate iterator for src-permuted tile decomposition -//! @param[in] __src_perm_src_strides Src-permuted source strides for loading -//! @param[in] __tile_src_perm_smem_strides Src-permuted shared memory strides for loading -//! @param[in] __tile_dst_perm_iter Coordinate iterator for dst-permuted tile decomposition -//! @param[in] __dst_perm_dst_strides Dst-permuted destination strides for storing -//! @param[in] __tile_dst_smem_strides Dst-permuted shared memory strides for storing -//! @param[in] __dst_strides Per-dimension destination strides for partial tiles -//! @param[in] __tile_total_size Total number of elements in one tile -//! @param[in] __tile_sizes Per-dimension tile extents -//! @param[in] __extents Per-dimension tensor extents (for partial-tile bounds) -//! @param[in] __src_strides Per-dimension source strides (for partial-tile access) -template -__global__ void __copy_shared_mem_kernel( - _CCCL_GRID_CONSTANT const _Config __config, - const _TpSrc* _CCCL_RESTRICT __src_ptr, - _CCCL_GRID_CONSTANT const _SrcAccessor __src_accessor, - _TpDst* _CCCL_RESTRICT __dst_ptr, - _CCCL_GRID_CONSTANT const _DstAccessor __dst_accessor, - _CCCL_GRID_CONSTANT const __tensor_coord_iterator<_ExtentT, _MaxRankUZ> __grid_iter, - _CCCL_GRID_CONSTANT const ::cuda::std::array<_StrideTIn, _MaxRankUZ> __grid_tile_src_strides, - _CCCL_GRID_CONSTANT const ::cuda::std::array<_StrideTOut, _MaxRankUZ> __grid_tile_dst_strides, - _CCCL_GRID_CONSTANT const __tensor_coord_iterator<__tile_extent_t, _MaxRankUZ> __tile_perm_iter, - _CCCL_GRID_CONSTANT const ::cuda::std::array<_StrideTIn, _MaxRankUZ> __src_perm_src_strides, - _CCCL_GRID_CONSTANT const ::cuda::std::array<__tile_extent_t, _MaxRankUZ> __tile_src_perm_smem_strides, - _CCCL_GRID_CONSTANT const __tensor_coord_iterator<__tile_extent_t, _MaxRankUZ> __tile_dst_perm_iter, - _CCCL_GRID_CONSTANT const ::cuda::std::array<_StrideTOut, _MaxRankUZ> __dst_perm_dst_strides, - _CCCL_GRID_CONSTANT const ::cuda::std::array<__tile_extent_t, _MaxRankUZ> __tile_dst_perm_smem_strides, - _CCCL_GRID_CONSTANT const ::cuda::std::array<_StrideTOut, _MaxRankUZ> __dst_strides, - _CCCL_GRID_CONSTANT const int __tile_total_size, - _CCCL_GRID_CONSTANT const ::cuda::std::array<__tile_extent_t, _MaxRankUZ> __tile_sizes, - _CCCL_GRID_CONSTANT const ::cuda::std::array<_ExtentT, _MaxRankUZ> __extents, - _CCCL_GRID_CONSTANT const ::cuda::std::array<_StrideTIn, _MaxRankUZ> __src_strides) -{ - constexpr auto __max_rank = int{_MaxRankUZ}; - // Grid tile decomposition: map linearized block index to src/dst base offsets - // __grid_coords: linear tile index -> multi-dimensional coordinates (array) - const auto __grid_index = ::cuda::block.index_as<_ExtentT>(::cuda::grid).x; - const auto __grid_coords = __grid_iter(__grid_index); - - { - _StrideTIn __src_base = 0; - _StrideTOut __dst_base = 0; - _CCCL_PRAGMA_UNROLL_FULL() - for (int __k = 0; __k < __max_rank; ++__k) - { - __src_base += static_cast<_StrideTIn>(__grid_coords[__k]) * __grid_tile_src_strides[__k]; - __dst_base += static_cast<_StrideTOut>(__grid_coords[__k]) * __grid_tile_dst_strides[__k]; - } - __src_ptr += __src_base; - __dst_ptr += __dst_base; - } - - // Partial tile detection: is the current tile full or partial? - bool __is_full_tile = true; - _CCCL_PRAGMA_UNROLL_FULL() - for (int __k = 0; __k < __max_rank; ++__k) - { - const auto __block_start = __grid_coords[__k] * __tile_sizes[__k]; - if (__block_start + __tile_sizes[__k] > __extents[__k]) - { - __is_full_tile = false; - break; - } - } - - // Dispatch to Full-tile or Boundary case - const auto __tid = ::cuda::gpu_thread.rank_as(::cuda::block, __config); - const auto __block_stride = ::cuda::gpu_thread.count_as(::cuda::block, __config); - using __partial_tensor_src = __partial_tensor; - using __partial_tensor_dst = __partial_tensor<_TpDst, _StrideTOut, _MaxRankUZ, _DstAccessor>; - - //-------------------------------------------------------------------------------------------------------------------- - // Full-tile shared-memory transpose - if (__is_full_tile) - { - using _Tp = ::cuda::std::remove_cv_t<_TpSrc>; - using __partial_tensor_smem = - __partial_tensor<_Tp, __tile_extent_t, _MaxRankUZ, ::cuda::std::default_accessor<_Tp>>; - - extern __shared__ char __smem_bytes[]; - auto* __smem = reinterpret_cast<_Tp*>(__smem_bytes); - - // (1) load src to shared memory by using the src/tile-permuted ordering - const __partial_tensor_src __src_tensor{__src_ptr, __src_perm_src_strides, __src_accessor}; - const __partial_tensor_smem __smem_tensor{ - __smem, __tile_src_perm_smem_strides, ::cuda::std::default_accessor<_Tp>{}}; - - for (auto __i = __tid; __i < __tile_total_size; __i += __block_stride) - { - const auto __coords = __tile_perm_iter(__i); - const auto __raw_offset = __smem_tensor.__offset(__coords); - const auto __swizzled_offset = ::cuda::experimental::__smem_offset<_UseXorSwizzle>(__raw_offset); - __smem[__swizzled_offset] = __src_tensor(__coords); - } - __syncthreads(); - - // (2) store from shared memory to destination by using the dst/tile-permuted ordering - const __partial_tensor_dst __dst_tensor{__dst_ptr, __dst_perm_dst_strides, __dst_accessor}; - const __partial_tensor_smem __smem_dst_tensor{ - __smem, __tile_dst_perm_smem_strides, ::cuda::std::default_accessor<_Tp>{}}; - - for (auto __i = __tid; __i < __tile_total_size; __i += __block_stride) - { - const auto __coords = __tile_dst_perm_iter(__i); - const auto __raw_offset = __smem_dst_tensor.__offset(__coords); - const auto __swizzled_offset = ::cuda::experimental::__smem_offset<_UseXorSwizzle>(__raw_offset); - __dst_tensor(__coords) = __smem[__swizzled_offset]; - } - } - - //-------------------------------------------------------------------------------------------------------------------- - // Boundary direct-copy (no shared memory) - else - { - using __uextent_t = ::cuda::std::make_unsigned_t<_ExtentT>; - const __partial_tensor_src __src_tensor{__src_ptr, __src_strides, __src_accessor}; - const __partial_tensor_dst __dst_tensor{__dst_ptr, __dst_strides, __dst_accessor}; - - // Find the partial tile sizes and total number of elements - ::cuda::std::array<__tile_extent_t, __max_rank> __partial_tile_sizes{}; - int __partial_tile_total = 1; - _CCCL_PRAGMA_UNROLL_FULL() - for (int __k = 0; __k < __max_rank; ++__k) - { - const auto __block_start = static_cast<__uextent_t>(__grid_coords[__k] * __tile_sizes[__k]); - const auto __diff = static_cast<__tile_extent_t>(__extents[__k] - __block_start); - __partial_tile_sizes[__k] = ::cuda::std::min(__tile_sizes[__k], __diff); - __partial_tile_total *= __partial_tile_sizes[__k]; - } - - // map the linear index to the multi-dimensional coordinates and copy the elements - for (auto __i = __tid; __i < __partial_tile_total; __i += __block_stride) - { - __tile_extent_t __linear = __i; - ::cuda::std::array<__tile_extent_t, __max_rank> __coords; - _CCCL_PRAGMA_UNROLL_FULL() - for (int __k = 0; __k < __max_rank; ++__k) - { - __coords[__k] = __linear % __partial_tile_sizes[__k]; - __linear /= __partial_tile_sizes[__k]; - } - __dst_tensor(__coords) = __src_tensor(__coords); - } - } -} - -#if !_CCCL_COMPILER(NVRTC) - -//! @brief Launch the shared-memory tiled transpose kernel. -//! -//! Precomputes the source/destination-coalesced permutations and tile shapes, constructs coordinate iterators, then -//! launches one block per tile. -//! -//! @pre `__src.__rank >= 2` -//! -//! @param[in] __src Source raw tensor descriptor -//! @param[out] __dst Destination raw tensor descriptor -//! @param[in] __stream CUDA stream for asynchronous execution -//! @param[in] __src_accessor Accessor for reading source elements -//! @param[in] __dst_accessor Accessor for writing destination elements -template -_CCCL_HOST_API void __launch_copy_shared_mem_kernel( - const __raw_tensor<_ExtentT, _StrideTIn, _TpIn, _MaxRank>& __src, - const __raw_tensor<_ExtentT, _StrideTOut, _TpOut, _MaxRank>& __dst, - ::cuda::stream_ref __stream, - const _SrcAccessor& __src_accessor = {}, - const _DstAccessor& __dst_accessor = {}) -{ - namespace cudax = ::cuda::experimental; - using ::cuda::std::size_t; - _CCCL_ASSERT(__src.__rank >= 2, "Rank must be at least 2 for shared memory transpose"); - - const auto __tiling = cudax::__find_shared_mem_tiling<_TpIn>(__src, __dst); - const auto __tile_sizes = __tiling.__tile_sizes; - const auto __rank = __src.__rank; - const auto __tile_total_size = __tiling.__tile_total_size; - - //-------------------------------------------------------------------------------------------------------------------- - // Find the grid size (number of blocks) and strides for block index decomposition - ::cuda::std::array<_ExtentT, _MaxRank> __grid_tile_sizes{}; - ::cuda::std::array<_StrideTIn, _MaxRank> __grid_tile_src_strides{}; - ::cuda::std::array<_StrideTOut, _MaxRank> __grid_tile_dst_strides{}; - _ExtentT __grid_size = 1; - for (size_t __i = 0; __i < __rank; ++__i) - { - __grid_tile_sizes[__i] = ::cuda::ceil_div(__src.__extents[__i], static_cast<_ExtentT>(__tile_sizes[__i])); - __grid_tile_src_strides[__i] = static_cast<_StrideTIn>(__tile_sizes[__i]) * __src.__strides[__i]; - __grid_tile_dst_strides[__i] = static_cast<_StrideTOut>(__tile_sizes[__i]) * __dst.__strides[__i]; - __grid_size *= __grid_tile_sizes[__i]; - } - for (size_t __i = __rank; __i < _MaxRank; ++__i) - { - __grid_tile_sizes[__i] = 1; - } - - //-------------------------------------------------------------------------------------------------------------------- - // Reordered arrays for loading src and storing dst based on coalesced permutations - ::cuda::std::array<_StrideTIn, _MaxRank> __src_perm_src_strides{}; - ::cuda::std::array<_StrideTOut, _MaxRank> __dst_perm_dst_strides{}; - ::cuda::std::array<__tile_extent_t, _MaxRank> __tile_src_perm_sizes{}; - ::cuda::std::array<__tile_extent_t, _MaxRank> __tile_dst_perm_sizes{}; - ::cuda::std::array<__tile_extent_t, _MaxRank> __tile_src_perm_smem_strides{}; - ::cuda::std::array<__tile_extent_t, _MaxRank> __tile_dst_perm_smem_strides{}; - ::cuda::std::array<__tile_extent_t, _MaxRank> __canonical_strides{}; - __canonical_strides[0] = 1; - for (size_t __i = 1; __i < __rank; ++__i) - { - __canonical_strides[__i] = __canonical_strides[__i - 1] * __tile_sizes[__i - 1]; - } - for (size_t __i = 0; __i < __rank; ++__i) - { - const auto __p = __tiling.__src_perm[__i]; - __tile_src_perm_sizes[__i] = __tile_sizes[__p]; - __src_perm_src_strides[__i] = __src.__strides[__p]; - __tile_src_perm_smem_strides[__i] = __canonical_strides[__p]; - - const auto __q = __tiling.__dst_perm[__i]; - __tile_dst_perm_sizes[__i] = __tile_sizes[__q]; - __dst_perm_dst_strides[__i] = __dst.__strides[__q]; - __tile_dst_perm_smem_strides[__i] = __canonical_strides[__q]; - } - for (size_t __i = __rank; __i < _MaxRank; ++__i) - { - __tile_src_perm_sizes[__i] = 1; - __tile_dst_perm_sizes[__i] = 1; - } - - //-------------------------------------------------------------------------------------------------------------------- - // Construct coordinate iterators on the host (precomputed fast modulo/division) - // namely, given a linear index, compute the multi-dimensional coordinates - const __tensor_coord_iterator<_ExtentT, _MaxRank> __grid_iter{__grid_tile_sizes}; // grid tile index - const __tensor_coord_iterator<__tile_extent_t, _MaxRank> __tile_perm_iter{__tile_src_perm_sizes}; // src -> shared - // memory - const __tensor_coord_iterator<__tile_extent_t, _MaxRank> __tile_dst_perm_iter{__tile_dst_perm_sizes}; // shared memory - // -> dst - - //-------------------------------------------------------------------------------------------------------------------- - // Launch the kernel - using __value_type = ::cuda::std::remove_cv_t<_TpIn>; - const int __thread_block_size = cudax::__find_thread_block_size(__tile_total_size * sizeof(__value_type)); - - const auto __config = ::cuda::make_config( - ::cuda::block_dims(__thread_block_size), - ::cuda::grid_dims(__grid_size), - ::cuda::dynamic_shared_memory<__value_type[]>(__tile_total_size)); - - if (__tiling.__use_xor_swizzle) - { - const auto __kernel = cudax::__copy_shared_mem_kernel< - true, - decltype(__config), - _MaxRank, - _TpIn, - _TpOut, - _SrcAccessor, - _DstAccessor, - _ExtentT, - _StrideTIn, - _StrideTOut>; - - ::cuda::launch( - __stream, - __config, - __kernel, - __src.__data, - __src_accessor, - __dst.__data, - __dst_accessor, - __grid_iter, - __grid_tile_src_strides, - __grid_tile_dst_strides, - __tile_perm_iter, - __src_perm_src_strides, - __tile_src_perm_smem_strides, - __tile_dst_perm_iter, - __dst_perm_dst_strides, - __tile_dst_perm_smem_strides, - __dst.__strides, - static_cast(__tile_total_size), - __tile_sizes, - __dst.__extents, - __src.__strides); - } - else - { - const auto __kernel = cudax::__copy_shared_mem_kernel< - false, - decltype(__config), - _MaxRank, - _TpIn, - _TpOut, - _SrcAccessor, - _DstAccessor, - _ExtentT, - _StrideTIn, - _StrideTOut>; - - ::cuda::launch( - __stream, - __config, - __kernel, - __src.__data, - __src_accessor, - __dst.__data, - __dst_accessor, - __grid_iter, - __grid_tile_src_strides, - __grid_tile_dst_strides, - __tile_perm_iter, - __src_perm_src_strides, - __tile_src_perm_smem_strides, - __tile_dst_perm_iter, - __dst_perm_dst_strides, - __tile_dst_perm_smem_strides, - __dst.__strides, - static_cast(__tile_total_size), - __tile_sizes, - __dst.__extents, - __src.__strides); - } -} - -#endif // !_CCCL_COMPILER(NVRTC) -} // namespace cuda::experimental - -#include - -#endif // _CUDAX__COPY_COPY_SHARED_MEMORY_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy/copy_shared_memory_utils.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy/copy_shared_memory_utils.cuh deleted file mode 100644 index 8a433235..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy/copy_shared_memory_utils.cuh +++ /dev/null @@ -1,293 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX__COPY_COPY_SHARED_MEMORY_UTILS_H -#define _CUDAX__COPY_COPY_SHARED_MEMORY_UTILS_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#if !_CCCL_COMPILER(NVRTC) - -# include -# include -# include -# include -# include -# include -# include -# include -# include - -# include -# include - -# include - -namespace cuda::experimental -{ -//! Maximum tensor rank for which the shared-memory transpose kernel is instantiated. Higher ranks cause excessive -//! register pressure (many rank-sized arrays and fully-unrolled loops). -inline constexpr ::cuda::std::size_t __max_shared_mem_kernel_rank = 8; - -//! A tile size is always representable by an unsigned integer. -using __tile_extent_t = unsigned; - -//! @brief Copy a raw tensor descriptor into one with a narrower static maximum rank. -//! -//! @param[in] __tensor Raw tensor descriptor with dynamic rank matching _RankOut -//! @return Raw tensor descriptor with _RankOut as its static maximum rank -template <::cuda::std::size_t _RankOut, typename _ExtentT, typename _StrideT, typename _Tp, ::cuda::std::size_t _MaxRank> -[[nodiscard]] _CCCL_HOST_API __raw_tensor<_ExtentT, _StrideT, _Tp, _RankOut> -__narrow_raw_tensor_rank(const __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>& __tensor) noexcept -{ - _CCCL_ASSERT(__tensor.__rank == _RankOut, "tensor rank must match the narrowed static rank"); - __raw_tensor<_ExtentT, _StrideT, _Tp, _RankOut> __result{__tensor.__data, _RankOut, {}, {}}; - for (::cuda::std::size_t __i = 0; __i < _RankOut; ++__i) - { - __result.__extents[__i] = __tensor.__extents[__i]; - __result.__strides[__i] = __tensor.__strides[__i]; - } - return __result; -} - -//! @brief Count the number of leading contiguous dimensions in a raw tensor. -//! -//! Starting from dimension 0, counts consecutive dimensions where `stride[0] == 1` and `stride[i] == stride[i-1] * -//! extent[i-1]` for each subsequent dimension. -//! -//! @param[in] __tensor Raw tensor descriptor -//! @return Number of leading contiguous dimensions (0 if stride[0] != 1 or rank is 0) -template -[[nodiscard]] _CCCL_HOST_API ::cuda::std::size_t -__num_contiguous_dimensions(const __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>& __tensor) noexcept -{ - using __rank_t = typename __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>::__rank_t; - if (__tensor.__rank == 0 || __tensor.__strides[0] != 1) - { - return 0; - } - __rank_t __count = 1; - auto __expected_stride = static_cast<_StrideT>(__tensor.__extents[0]); - for (__rank_t __i = 1; __i < __tensor.__rank; ++__i) - { - if (__tensor.__strides[__i] != __expected_stride) - { - break; - } - __expected_stride *= static_cast<_StrideT>(__tensor.__extents[__i]); - ++__count; - } - return __count; -} - -//! @brief Return a device_ref for the current CUDA device. -//! -//! @return Device reference for the active CUDA context's device -[[nodiscard]] _CCCL_HOST_API inline ::cuda::device_ref __current_device() noexcept -{ - const auto __dev_id = ::cuda::__driver::__cudevice_to_ordinal(::cuda::__driver::__ctxGetDevice()); - return ::cuda::devices[__dev_id]; -} - -//! Maximum extent of a single tile dimension, set to the warp size so that the innermost tile dimension maps to a -//! full warp of coalesced accesses. -inline constexpr size_t __max_tile_size = 32; - -// The structure holds the tiling information to optimize the transpose (shared-memory) kernel. -// - __tile_sizes: the size of each tile dimension in shared-memory -// - __src_perm: the permutation of the source dimensions (copy to shared-memory) -// - __dst_perm: the permutation of the destination dimensions (copy from shared-memory) -// - __tile_total_size: the total size of the tile in shared-memory -// - __active_tile_dims: the number of dimensions covered by the tile -// - __active_32_dims: the number of tile dimensions with extent __max_tile_size -// - __is_valid: true if the tiling is valid -// - __use_xor_swizzle: true if the XOR swizzle is used -template <::cuda::std::size_t _MaxRank> -struct __shared_mem_tiling_result -{ - ::cuda::std::array<__tile_extent_t, _MaxRank> __tile_sizes{}; - ::cuda::std::array<::cuda::std::size_t, _MaxRank> __src_perm{}; - ::cuda::std::array<::cuda::std::size_t, _MaxRank> __dst_perm{}; - ::cuda::std::size_t __tile_total_size = 1; - ::cuda::std::size_t __active_tile_dims = 0; - ::cuda::std::size_t __active_32_dims = 0; - bool __is_valid = false; - bool __use_xor_swizzle = false; -}; - -//! @brief Adds a contiguous stride-1 run from one tensor to the shared-memory tile. -//! -//! @param[in] __tensor Raw tensor descriptor used to find coalesced modes -//! @param[in] __perm Mode order to scan -//! @param[in,out] __result Shared-memory tiling result updated with selected tile sizes -//! @param[in] __max_shared_mem_bytes Maximum shared-memory capacity for one tile -//! @return Number of coalesced elements covered by this tensor's selected tile run -template -[[nodiscard]] _CCCL_HOST_API ::cuda::std::size_t __add_coalesced_tile_run( - const __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>& __tensor, - const ::cuda::std::array<::cuda::std::size_t, _MaxRank>& __perm, - __shared_mem_tiling_result<_MaxRank>& __result, - ::cuda::std::size_t __max_shared_mem_bytes) noexcept -{ - using ::cuda::std::size_t; - size_t __coalesced_tile_size = 1; - _StrideT __expected_stride = 1; - - for (size_t __i = 0; __i < __tensor.__rank; ++__i) - { - const auto __perm_i = __perm[__i]; - const auto __extent = static_cast(__tensor.__extents[__perm_i]); - const auto __stride = ::cuda::experimental::__abs_integer(__tensor.__strides[__perm_i]); - if (__stride != __expected_stride) // input tensor not contiguous - { - break; - } - - if (__result.__tile_sizes[__perm_i] == 1) // first time we see this dimension - { - const auto __tile_size = ::cuda::std::min(__extent, __max_tile_size); - const auto __tile_total_size_bytes = __result.__tile_total_size * __tile_size * sizeof(_SmemTp); - if (__tile_total_size_bytes > __max_shared_mem_bytes) - { - break; - } - // if the tile fits in shared-memory, update the result - __result.__tile_sizes[__perm_i] = static_cast<__tile_extent_t>(__tile_size); - __result.__tile_total_size *= __tile_size; - ++__result.__active_tile_dims; - if (__tile_size == __max_tile_size) - { - ++__result.__active_32_dims; - } - } - __coalesced_tile_size *= __result.__tile_sizes[__perm_i]; - __expected_stride *= static_cast<_StrideT>(__extent); - } - return __coalesced_tile_size; -} - -//! @brief Compute a source/destination-aware shared-memory tile. -//! -//! The selected tile spans coalesced dimensions from both layouts. This keeps the load phase ordered by source stride -//! and the store phase ordered by destination stride, without requiring either coalesced dimension to be mode 0. -//! -//! @param[in] __src Source raw tensor descriptor -//! @param[in] __dst Destination raw tensor descriptor -//! @return Shared-memory tiling decision and layout permutations -template -[[nodiscard]] _CCCL_HOST_API __shared_mem_tiling_result<_MaxRank> -__find_shared_mem_tiling(const __raw_tensor<_ExtentT, _StrideTIn, _TpSrc, _MaxRank>& __src, - const __raw_tensor<_ExtentT, _StrideTOut, _TpDst, _MaxRank>& __dst) noexcept -{ - using ::cuda::std::size_t; - __shared_mem_tiling_result<_MaxRank> __result{}; - // initialize the source and destination permutations and sort them by stride - for (size_t __i = 0; __i < _MaxRank; ++__i) - { - __result.__tile_sizes[__i] = 1; - __result.__src_perm[__i] = __i; - __result.__dst_perm[__i] = __i; - } - __result.__src_perm = ::cuda::experimental::__stride_order(__src); - __result.__dst_perm = ::cuda::experimental::__stride_order(__dst); - - const auto __current_dev = ::cuda::experimental::__current_device(); - const size_t __max_shared_mem_bytes = __current_dev.attribute<::cudaDevAttrMaxSharedMemoryPerBlock>(); - const auto __src_coalesced_tile_size = - ::cuda::experimental::__add_coalesced_tile_run<_TpIn>(__src, __result.__src_perm, __result, __max_shared_mem_bytes); - const auto __dst_coalesced_tile_size = - ::cuda::experimental::__add_coalesced_tile_run<_TpIn>(__dst, __result.__dst_perm, __result, __max_shared_mem_bytes); - - // If the tile total size is too small, or the coalescing is not useful on both sides, or the number of active tile - // dimensions is less than 2, return the result. - if (__result.__tile_total_size < __max_tile_size * 8 || __src_coalesced_tile_size < 2 || __dst_coalesced_tile_size < 2 - || __result.__active_tile_dims < 2) - { - return __result; - } - - // There must be enough blocks to keep the GPU busy (at least one full wave across all SMs). - const size_t __num_sms = __current_dev.attribute<::cudaDevAttrMultiProcessorCount>(); - size_t __num_tiles = 1; - for (size_t __r = 0; __r < __dst.__rank; ++__r) - { - const auto __extent = static_cast(__dst.__extents[__r]); - const auto __tile_size = static_cast(__result.__tile_sizes[__r]); - __num_tiles *= ::cuda::ceil_div(__extent, __tile_size); - } - if (__num_tiles < __num_sms) - { - return __result; - } - - __result.__is_valid = true; - // Shared memory swizzle makes sense only for 32-bit and 64-bit types. - __result.__use_xor_swizzle = (sizeof(_TpIn) == 4 || sizeof(_TpIn) == 8) // - && __result.__active_32_dims == 2; - return __result; -} - -//! @brief Decide whether the shared-memory tiled transpose kernel is profitable. -//! -//! @param[in] __src Source raw tensor descriptor -//! @param[in] __dst Destination raw tensor descriptor -//! @return true if the shared-memory kernel should be used -template -[[nodiscard]] _CCCL_HOST_API bool -__use_shared_mem_kernel(const __raw_tensor<_ExtentT, _StrideTIn, _TpIn, _MaxRank>& __src, - const __raw_tensor<_ExtentT, _StrideTOut, _TpOut, _MaxRank>& __dst) noexcept -{ - return ::cuda::experimental::__find_shared_mem_tiling<_TpIn>(__src, __dst).__is_valid; -} - -//! @brief Compute the thread block size for the shared-memory kernel. -//! -//! Balances occupancy by dividing the SM threads across as many blocks as the shared memory allows, then caps at -//! the device maximum. -//! -//! @param[in] __tile_total_bytes Shared memory required for one tile in bytes -//! @return Thread block size -[[nodiscard]] _CCCL_HOST_API inline int __find_thread_block_size(::cuda::std::size_t __tile_total_bytes) noexcept -{ - using ::cuda::std::size_t; - const auto __dev = ::cuda::experimental::__current_device(); - const size_t __total_sm_threads = __dev.attribute<::cudaDevAttrMaxThreadsPerMultiProcessor>(); - const size_t __max_thread_block_size = __dev.attribute<::cudaDevAttrMaxThreadsPerBlock>(); - const size_t __total_shared_mem_bytes = __dev.attribute<::cudaDevAttrMaxSharedMemoryPerMultiprocessor>(); - const auto __num_blocks_per_sm = __total_shared_mem_bytes / __tile_total_bytes; - const auto __thread_block_size = ::cuda::std::min(__total_sm_threads / __num_blocks_per_sm, __max_thread_block_size); - const auto __thread_block_size32 = ::cuda::round_up(__thread_block_size, /*warp size=*/size_t{32}); - return static_cast(__thread_block_size32); -} -} // namespace cuda::experimental - -# include - -#endif // !_CCCL_COMPILER(NVRTC) -#endif // _CUDAX__COPY_COPY_SHARED_MEMORY_UTILS_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy/dispatch_by_vector.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy/dispatch_by_vector.cuh deleted file mode 100644 index 4f6420aa..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy/dispatch_by_vector.cuh +++ /dev/null @@ -1,149 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX__COPY_DISPATCH_BY_VECTOR_H -#define _CUDAX__COPY_DISPATCH_BY_VECTOR_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#if !_CCCL_COMPILER(NVRTC) - -# include -# include -# include - -# include -# include - -# include - -namespace cuda::experimental -{ -//! @brief Compute the maximum vector access width in bytes for a pair of raw tensors. -//! -//! Takes the minimum of the source alignment, destination alignment, and the GPU architecture's -//! maximum vector width. -//! -//! @param[in] __src Source raw tensor -//! @param[in] __dst Destination raw tensor -//! @return Maximum safe vector access width in bytes -template -[[nodiscard]] _CCCL_HOST_API ::cuda::std::size_t -__vector_size_bytes(const __raw_tensor<_SrcExtentT, _SrcStrideT, _TpSrc, _MaxRank>& __src, - const __raw_tensor<_DstExtentT, _DstStrideT, _TpDst, _MaxRank>& __dst) noexcept -{ - return ::cuda::std::min( - {::cuda::experimental::__max_alignment(__src), - ::cuda::experimental::__max_alignment(__dst), - ::cuda::experimental::__max_gpu_arch_vector_size()}); -} - -template -constexpr auto __const_vector_size = ::cuda::std::integral_constant{}; - -//! @brief Dispatch a copy operation with the optimal vectorized element type. -//! -//! Computes the maximum safe vector width from the source and destination tensors, reshapes both -//! tensors to that vector type via @ref __reshape_vectorized, and invokes @p __op with the reshaped tensors. -//! -//! @param[in] __src Source raw tensor descriptor -//! @param[in] __dst Destination raw tensor descriptor -//! @param[in] __op Callable invoked with the reshaped source and destination tensors -template -_CCCL_HOST_API void __dispatch_by_vector_size( - const __raw_tensor<_ExtentT, _StrideTIn, _TpIn, _Rank>& __src, - const __raw_tensor<_ExtentT, _StrideTOut, _TpOut, _Rank>& __dst, - _Op __op) noexcept -{ - namespace cudax = ::cuda::experimental; - const auto __call_vectorized = [&](auto __const_vector_size) { - const auto __src_recast = cudax::__reshape_vectorized<__const_vector_size>(__src); - const auto __dst_recast = cudax::__reshape_vectorized<__const_vector_size>(__dst); - __op(__src_recast, __dst_recast); - }; - const auto __vector_size_bytes = cudax::__vector_size_bytes(__src, __dst); -// 32-bytes aligned vector types have been introduced in CTK 13.0 -# if _CCCL_CTK_AT_LEAST(13, 0) - static_assert(sizeof(_TpIn) <= 32); - if constexpr (sizeof(_TpIn) <= 32) - { - if (__vector_size_bytes == 32) - { - __call_vectorized(__const_vector_size<32>); - return; - } - } -# else // ^^^ _CCCL_CTK_AT_LEAST(13, 0) ^^^ / vvv _CCCL_CTK_BELOW(13, 0) vvv - static_assert(sizeof(_TpIn) <= 16); -# endif // _CCCL_CTK_AT_LEAST(13, 0) - if constexpr (sizeof(_TpIn) <= 16) - { - if (__vector_size_bytes == 16) - { - __call_vectorized(__const_vector_size<16>); - return; - } - } - if constexpr (sizeof(_TpIn) <= 8) - { - if (__vector_size_bytes == 8) - { - __call_vectorized(__const_vector_size<8>); - return; - } - } - if constexpr (sizeof(_TpIn) <= 4) - { - if (__vector_size_bytes == 4) - { - __call_vectorized(__const_vector_size<4>); - return; - } - } - if constexpr (sizeof(_TpIn) <= 2) - { - if (__vector_size_bytes == 2) - { - __call_vectorized(__const_vector_size<2>); - return; - } - } - if constexpr (sizeof(_TpIn) <= 1) - { - __call_vectorized(__const_vector_size<1>); - } - // no fallthrough (sizeof(T) is never 0) -} -} // namespace cuda::experimental - -# include - -#endif // !_CCCL_COMPILER(NVRTC) -#endif // _CUDAX__COPY_DISPATCH_BY_VECTOR_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy/mdspan_d2d.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy/mdspan_d2d.cuh deleted file mode 100644 index b4bf38f5..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy/mdspan_d2d.cuh +++ /dev/null @@ -1,266 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX__COPY_MDSPAN_D2D_H -#define _CUDAX__COPY_MDSPAN_D2D_H - -#include - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#if !_CCCL_COMPILER(NVRTC) - -# include - -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include - -# include -# include -# include -# include -# include -# include -# include -# include - -# include - -namespace cuda::experimental -{ -//! @brief Copy elements between two device mdspans. -//! -//! Validates preconditions, converts mdspans to raw tensor descriptors, simplifies the paired layout -//! (sort, flip negative strides, coalesce), then dispatches either a vectorized contiguous kernel or a -//! strided element-wise kernel. -//! -//! @param[in] __src Source device mdspan -//! @param[out] __dst Destination device mdspan -//! @param[in] __stream CUDA stream for the asynchronous transfer -template -_CCCL_HOST_API void copy(::cuda::device_mdspan<_TpIn, _ExtentsIn, _LayoutPolicyIn, _AccessorPolicyIn> __src, - ::cuda::device_mdspan<_TpOut, _ExtentsOut, _LayoutPolicyOut, _AccessorPolicyOut> __dst, - ::cuda::stream_ref __stream) -{ - namespace cudax = ::cuda::experimental; - static_assert(::cuda::std::is_convertible_v<_TpIn, _TpOut>, "TpIn must be convertible to TpOut"); - static_assert(!::cuda::std::is_const_v<_TpOut>, "TpOut must not be const"); - static_assert(::cuda::__is_cuda_mdspan_layout_v<_LayoutPolicyIn>, - "LayoutPolicyIn must be a predefined layout policy"); - static_assert(::cuda::__is_cuda_mdspan_layout_v<_LayoutPolicyOut>, - "LayoutPolicyOut must be a predefined layout policy"); - - if (__src.size() != __dst.size()) - { - _CCCL_THROW(::std::invalid_argument, "mdspans must have the same size"); - } - const auto __tensor_size = __src.size(); - if (__tensor_size == 0) - { - return; - } - if (__src.data_handle() == nullptr || __dst.data_handle() == nullptr) - { - _CCCL_THROW(::std::invalid_argument, "mdspan data handle must not be nullptr"); - } - if (!::cuda::std::is_sufficiently_aligned(__src.data_handle())) - { - _CCCL_THROW(::std::invalid_argument, "source mdspan must be sufficiently aligned"); - } - if (!::cuda::std::is_sufficiently_aligned(__dst.data_handle())) - { - _CCCL_THROW(::std::invalid_argument, "destination mdspan must be sufficiently aligned"); - } - if (cudax::__has_interleaved_stride_order(__dst)) - { - _CCCL_THROW(::std::invalid_argument, "destination mdspan must not have interleaved stride order"); - } - if (cudax::__may_overlap(__src, __dst)) - { - _CCCL_THROW(::std::invalid_argument, "mdspans must not overlap in memory"); - } - - using __default_accessor_in = ::cuda::std::default_accessor<_TpIn>; - using __default_accessor_out = ::cuda::std::default_accessor<_TpOut>; - constexpr bool __have_default_accessors = - ::cuda::std::is_convertible_v<_AccessorPolicyIn, __default_accessor_in> - && ::cuda::std::is_convertible_v<_AccessorPolicyOut, __default_accessor_out>; - constexpr bool __are_byte_copyable = - ::cuda::std::is_same_v<::cuda::std::remove_cv_t<_TpIn>, ::cuda::std::remove_cv_t<_TpOut>> - && ::cuda::is_trivially_copyable_v<_TpIn> // - && __have_default_accessors; - - if (__tensor_size == 1 && __are_byte_copyable) - { - auto __src_ptr = __src.data_handle(); - auto __dst_ptr = __dst.data_handle(); - if constexpr (::cuda::__is_layout_stride_relaxed_v<_LayoutPolicyIn>) - { - __src_ptr += __src.mapping().offset(); - } - if constexpr (::cuda::__is_layout_stride_relaxed_v<_LayoutPolicyOut>) - { - __dst_ptr += __dst.mapping().offset(); - } - ::cuda::__driver::__memcpyAsync(__dst_ptr, __src_ptr, sizeof(_TpIn), __stream.get()); - return; - } - - // rank == 0 for both tensors is already handled above -> their size is exactly 1 - if constexpr (_ExtentsIn::rank() > 0 && _ExtentsOut::rank() > 0) - { - // use the most efficient type for device code - using __src_extent_t = ::cuda::std::common_type_t; - using __dst_extent_t = ::cuda::std::common_type_t; - using __common_extent_t = - ::cuda::std::conditional_t<(sizeof(__src_extent_t) < sizeof(__dst_extent_t)), __src_extent_t, __dst_extent_t>; - using __src_stride_t = - ::cuda::std::common_type_t, int>; - using __dst_stride_t = - ::cuda::std::common_type_t, int>; - - constexpr auto __max_rank = ::cuda::std::max(_ExtentsIn::rank(), _ExtentsOut::rank()); - const auto __src_raw = cudax::__to_raw_tensor<__common_extent_t, __src_stride_t, __max_rank>(__src); - const auto __dst_raw = cudax::__to_raw_tensor<__common_extent_t, __dst_stride_t, __max_rank>(__dst); - if (!cudax::__same_extents(__src_raw, __dst_raw)) - { - _CCCL_THROW(::std::invalid_argument, "mdspans must have the same extents (after removing singleton dimensions)"); - } - - auto __src_simplified = __src_raw; - auto __dst_simplified = __dst_raw; - cudax::__sort_by_stride_paired(__src_simplified, __dst_simplified); - cudax::__flip_negative_strides_paired(__src_simplified, __dst_simplified); - cudax::__coalesce_paired(__src_simplified, __dst_simplified); - const bool __both_stride1 = (__src_simplified.__strides[0] == 1) && (__dst_simplified.__strides[0] == 1); - const auto __tile_size = __both_stride1 ? __src_simplified.__extents[0] : 1; - const auto __src_normalized = (__tile_size > 1) ? __src_simplified : cudax::__reverse_modes(__src_raw); - const auto __dst_normalized = (__tile_size > 1) ? __dst_simplified : cudax::__reverse_modes(__dst_raw); - - _CCCL_ASSERT(__tensor_size % __tile_size == 0, "tensor size must be divisible by tile size"); - const auto __inner_extent_bytes = __src_normalized.__extents[0] * sizeof(_TpIn); - - // check the preconditions for the vectorized case - constexpr bool __are_vectorizable_copy = - sizeof(_TpIn) <= __max_vector_access && ::cuda::is_power_of_two(sizeof(_TpIn)) && __are_byte_copyable; - - // (1) contiguous case - if constexpr (__have_default_accessors) - { - if (static_cast<::cuda::std::size_t>(__tile_size) == __tensor_size) - { - _CCCL_TRY_CUDA_API( - CUB_NS_QUALIFIER::DeviceTransform::Transform, - "cub::DeviceTransform::Transform failed", - __src_simplified.__data, - __dst_simplified.__data, - __tensor_size, - ::cuda::proclaim_copyable_arguments(::cuda::std::identity{}), - __stream.get()); - return; - } - } - // (2) inner size is large - if (__both_stride1 && __inner_extent_bytes >= cudax::__bytes_in_flight()) - { - // (2a) vectorized case - if constexpr (__are_vectorizable_copy) - { - const auto __op = [__stream](const auto& __src, const auto& __dst) { - cudax::__launch_copy_contiguous_kernel(__src, __dst, __stream); - }; - cudax::__dispatch_by_vector_size(__src_normalized, __dst_normalized, __op); - } - // (2b) non-vectorized case but inner size is large enough to use the contiguous kernel - else - { - cudax::__launch_copy_contiguous_kernel( - __src_normalized, __dst_normalized, __stream, __src.accessor(), __dst.accessor()); - } - return; - } - // (3) inner size is not large -> try vectorized case - if constexpr (__are_vectorizable_copy) - { - if (__both_stride1) - { - const auto __op = [__stream](const auto& __src, const auto& __dst) { - cudax::__copy_optimized(__src, __dst, cudax::__total_size(__src), __stream); - }; - cudax::__dispatch_by_vector_size(__src_normalized, __dst_normalized, __op); - return; - } - } - // (4) transpose case (rank capped to avoid excessive register pressure in the kernel) - if constexpr (__max_rank >= 2 && __max_rank <= cudax::__max_shared_mem_kernel_rank) - { - if (__src_simplified.__rank == 2) // Optimize when the actual rank is 2 - { - const auto __src_rank2 = cudax::__narrow_raw_tensor_rank<2>(__src_simplified); - const auto __dst_rank2 = cudax::__narrow_raw_tensor_rank<2>(__dst_simplified); - if (cudax::__use_shared_mem_kernel(__src_rank2, __dst_rank2)) - { - cudax::__launch_copy_shared_mem_kernel(__src_rank2, __dst_rank2, __stream, __src.accessor(), __dst.accessor()); - return; - } - } - if (cudax::__use_shared_mem_kernel(__src_simplified, __dst_simplified)) - { - cudax::__launch_copy_shared_mem_kernel( - __src_simplified, __dst_simplified, __stream, __src.accessor(), __dst.accessor()); - return; - } - } - // (5) generic case (fallback) - cudax::__copy_optimized( - __src_normalized, - __dst_normalized, - cudax::__total_size(__src_normalized), - __stream, - __src.accessor(), - __dst.accessor()); - } -} -} // namespace cuda::experimental - -# include - -#endif // !_CCCL_COMPILER(NVRTC) -#endif // _CUDAX__COPY_MDSPAN_D2D_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy/tensor_copy_utils.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy/tensor_copy_utils.cuh deleted file mode 100644 index 0846837c..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy/tensor_copy_utils.cuh +++ /dev/null @@ -1,192 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX__COPY_TENSOR_COPY_UTILS_H -#define _CUDAX__COPY_TENSOR_COPY_UTILS_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#if !_CCCL_COMPILER(NVRTC) - -# include -# include -# include -# include -# include -# include -# include -# include -# include - -# include -# include -# include - -# include - -namespace cuda::experimental -{ -//! @brief Compute the maximum vectorization width in bytes for a raw tensor. -//! -//! Expects mode 0 to be the contiguous mode (stride == 1), as established by -//! @ref __sort_by_stride_paired. Computes the largest power-of-two vector width such that: -//! - The pointer is aligned to that width. -//! - All non-contiguous strides (in bytes) are divisible by it. -//! - The contiguous mode's shape is divisible by the element count. -//! The result is capped at 16 bytes. If mode 0 is not contiguous, returns sizeof(_Tp). -//! -//! @pre `__tensor.__rank` is in [1, _MaxRank]. -//! @pre All shapes must be > 1 (no degenerate modes). -//! @pre Strides are sorted by @ref __sort_by_stride_paired (mode 0 has the smallest absolute stride). -//! -//! @param[in] __tensor Raw tensor with strides sorted by @ref __sort_by_stride_paired -//! @return Maximum safe vectorization width in bytes, in [sizeof(_Tp), 16] -template -[[nodiscard]] _CCCL_HOST_API ::cuda::std::size_t -__max_alignment(const __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>& __tensor) noexcept -{ - using ::cuda::std::size_t; - using __raw_tensor_t = __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>; - using __rank_t = typename __raw_tensor_t::__rank_t; - _CCCL_ASSERT(::cuda::in_range(__tensor.__rank, size_t{1}, _MaxRank), "Invalid tensor rank"); - if (__tensor.__strides[0] != 1) - { - return sizeof(_Tp); - } - // (1) pointer alignment - size_t __alignment = ::cuda::__ptr_alignment(__tensor.__data); - // (2) alignment over all strides - for (__rank_t __i = 0; __i < __tensor.__rank; ++__i) - { - const auto __stride = ::cuda::experimental::__abs_integer(__tensor.__strides[__i]); - if (__stride != 1) - { - const size_t __stride_bytes = static_cast(__stride) * sizeof(_Tp); - __alignment = ::cuda::std::gcd(__alignment, __stride_bytes); - } - } - _CCCL_ASSERT(__alignment % sizeof(_Tp) == 0, "alignment is not a multiple of the element size"); - // (3) Compute the number of items per vector over the contiguous mode - size_t __elem_alignment = __alignment / sizeof(_Tp); - __elem_alignment = ::cuda::std::gcd(__elem_alignment, static_cast(__tensor.__extents[0])); - return __elem_alignment * sizeof(_Tp); -} - -template <::cuda::std::size_t _VectorBytes, typename _Tp> -using __reshape_vector_type = - ::cuda::std::conditional_t<::cuda::std::is_const_v<_Tp>, - const ::cuda::experimental::__vector_access_t<_VectorBytes>, - ::cuda::experimental::__vector_access_t<_VectorBytes>>; - -//! @brief Reshape a raw tensor for vectorized access by widening the element type. -//! -//! @pre Mode 0 must be contiguous (stride == 1). -//! @pre The innermost extent (in bytes) must be divisible by @p _VectorBytes. -//! @pre All non-innermost strides must be divisible by the elements-per-vector ratio. -//! -//! @tparam _VectorBytes Target vector width in bytes -//! @param[in] __tensor Raw tensor with contiguous innermost mode -//! @return Raw tensor with element type replaced by the vector type and adjusted extents/strides -template <::cuda::std::size_t _VectorBytes, typename _ExtentT, typename _StrideT, typename _Tp, ::cuda::std::size_t _MaxRank> -[[nodiscard]] -_CCCL_HOST_API __raw_tensor<_ExtentT, _StrideT, __reshape_vector_type<_VectorBytes, _Tp>, _MaxRank> -__reshape_vectorized(const __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>& __tensor) noexcept -{ - using __vector_t = __reshape_vector_type<_VectorBytes, _Tp>; - using __rank_t = typename __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>::__rank_t; - static_assert(_VectorBytes % sizeof(_Tp) == 0, "vector size must be a multiple of element size"); - - constexpr auto __elems_per_vector = _VectorBytes / sizeof(_Tp); - _CCCL_ASSERT(__tensor.__strides[0] == 1, "innermost mode must be contiguous"); - _CCCL_ASSERT(__tensor.__extents[0] % __elems_per_vector == 0, - "innermost extent must be divisible by elements per vector"); - _CCCL_ASSERT(::cuda::std::is_sufficiently_aligned(__tensor.__data), - "tensor data is not sufficiently aligned to the extents and strides"); - - const auto __data = reinterpret_cast<__vector_t*>(__tensor.__data); - __raw_tensor<_ExtentT, _StrideT, __vector_t, _MaxRank> __result{ - __data, __tensor.__rank, __tensor.__extents, __tensor.__strides}; - __result.__extents[0] /= __elems_per_vector; - - for (__rank_t __i = 1; __i < __result.__rank; ++__i) - { - _CCCL_ASSERT(__result.__strides[__i] % _StrideT{__elems_per_vector} == 0, - "non-innermost strides must be divisible by elements per vector"); - __result.__strides[__i] /= _StrideT{__elems_per_vector}; - } - return __result; -} - -//! @brief Compute the total number of elements in a raw tensor. -//! -//! @param[in] __tensor Raw tensor descriptor -//! @return Product of all extents -template -[[nodiscard]] -_CCCL_HOST_API _ExtentT __total_size(const __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>& __tensor) noexcept -{ - using __raw_tensor_t = __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>; - using __rank_t = typename __raw_tensor_t::__rank_t; - _ExtentT __total_size = 1; - for (__rank_t __i = 0; __i < __tensor.__rank; ++__i) - { - __total_size *= __tensor.__extents[__i]; - } - return __total_size; -} - -//! @brief Conservative check whether two mdspans may access overlapping memory. -//! -//! Uses each mapping's @c required_span_size() to compute the half-open byte range -//! @c [data_handle, data_handle + required_span_size * sizeof(T)) and checks for intersection. -//! NOTE: the function doesn't check strict overlap for non-contiguous layouts, for example, the padding could be -//! between the two mdspans. -//! -//! Empty mdspans (size == 0) are considered non-overlapping. -//! -//! @param[in] __a First mdspan -//! @param[in] __b Second mdspan -//! @return true if the byte ranges of the two mdspans overlap -template -[[nodiscard]] _CCCL_HOST_API bool -__may_overlap(const ::cuda::std::mdspan<_Tp1, _Extents1, _LayoutPolicy1, _AccessorPolicy1>& __a, - const ::cuda::std::mdspan<_Tp2, _Extents2, _LayoutPolicy2, _AccessorPolicy2>& __b) noexcept -{ - if (__a.size() == 0 || __b.size() == 0) - { - return false; - } - const auto* __a_begin = reinterpret_cast(__a.data_handle()); - const auto* __b_begin = reinterpret_cast(__b.data_handle()); - const auto* __a_end = __a_begin + __a.mapping().required_span_size() * sizeof(_Tp1); - const auto* __b_end = __b_begin + __b.mapping().required_span_size() * sizeof(_Tp2); - return ::cuda::ranges_overlap(__a_begin, __a_end, __b_begin, __b_end); -} -} // namespace cuda::experimental - -# include - -#endif // !_CCCL_COMPILER(NVRTC) -#endif // _CUDAX__COPY_TENSOR_COPY_UTILS_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy/tensor_iterator.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy/tensor_iterator.cuh deleted file mode 100644 index 5dae1eab..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy/tensor_iterator.cuh +++ /dev/null @@ -1,170 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX__COPY_TENSOR_ITERATOR_H -#define _CUDAX__COPY_TENSOR_ITERATOR_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include - -#include - -namespace cuda::experimental -{ -/*********************************************************************************************************************** - * Fast Modulo/Division based on Precomputation - **********************************************************************************************************************/ - -template -[[nodiscard]] _CCCL_HOST_DEVICE_API ::cuda::std::array<::cuda::fast_mod_div<_ExtentT>, sizeof...(_Rp)> -__extents_fast_div_mod_impl(const ::cuda::std::array<_ExtentT, _Size>& __extents, - ::cuda::std::index_sequence<_Rp...> = {}) noexcept -{ - using __fast_mod_div_t = ::cuda::fast_mod_div<_ExtentT>; - using __array_t = ::cuda::std::array<__fast_mod_div_t, sizeof...(_Rp)>; - return __array_t{__fast_mod_div_t(__extents[_Rp])...}; -} - -//! @brief Precompute modulo/division for each array extent. -//! -//! @param[in] __extents Array of extents -//! @return Array of precomputed fast modulo/division objects -template -[[nodiscard]] _CCCL_HOST_DEVICE_API ::cuda::std::array<::cuda::fast_mod_div<_ExtentT>, _Size> -__extents_fast_div_mod(const ::cuda::std::array<_ExtentT, _Size>& __extents) noexcept -{ - using __seq_t = ::cuda::std::make_index_sequence<_Size>; - return ::cuda::experimental::__extents_fast_div_mod_impl(__extents, __seq_t{}); -} - -/*********************************************************************************************************************** - * Tensor Coordinate Iterator and Partial Tensor - **********************************************************************************************************************/ - -//! @brief Iterator that maps a linear tile index to a pointer into a strided raw tensor. -template -struct __tensor_coord_iterator -{ - using __unsigned_extent_t = ::cuda::std::make_unsigned_t<_ExtentT>; - using __fast_mod_div_t = ::cuda::fast_mod_div<__unsigned_extent_t>; - using __array_t = ::cuda::std::array<__fast_mod_div_t, _Rank>; - - __array_t __extents_; - - //! @brief Convert an array of _UExtentT elements to an array of _ExtentT elements. - //! - //! @param[in] __in_array Source array with elements of type _UExtentT - //! @return Array with elements statically cast to _ExtentT - template - [[nodiscard]] static _CCCL_HOST_API ::cuda::std::array<__unsigned_extent_t, _Rank> - __to_extent_array(const ::cuda::std::array<_UExtentT, _Rank>& __in_array) noexcept - { - ::cuda::std::array<__unsigned_extent_t, _Rank> __out_array{}; - for (::cuda::std::size_t __i = 0; __i < _Rank; ++__i) - { - __out_array[__i] = static_cast<__unsigned_extent_t>(__in_array[__i]); - } - return __out_array; - } - - //! @brief Constructs the iterator from tensor extents. - //! - //! @param[in] __extents Tensor extents (may be unsigned; converted to _ExtentT internally) - template - _CCCL_HOST_API explicit __tensor_coord_iterator(const ::cuda::std::array<_UExtentT, _Rank>& __extents) noexcept - : __extents_{::cuda::experimental::__extents_fast_div_mod(__to_extent_array(__extents))} - {} - - //! @brief Returns the multi-dimensional coordinates for the given linear index. - //! - //! @param[in] __index Linear tile index - //! @return Array of coordinates into the tensor - [[nodiscard]] _CCCL_HOST_DEVICE_API ::cuda::std::array<_ExtentT, _Rank> operator()(_ExtentT __index) const noexcept - { - if constexpr (_Rank == 1) - { - return ::cuda::std::array<_ExtentT, _Rank>{{__index}}; - } - else - { - // instead of computing the coordinate in parallel (index / prod(extent_i) % extent_i), we use a simpler and - // slower approach. This saves registers and makes the overall computation faster. - ::cuda::std::array<_ExtentT, _Rank> __coords{}; - auto __quotient = static_cast<__unsigned_extent_t>(__index); - _CCCL_PRAGMA_UNROLL_FULL() - for (int __i = 0; __i < int{_Rank} - 1; ++__i) - { - const auto __div_result = ::cuda::div(__quotient, __extents_[__i]); - __quotient = __div_result.first; - __coords[__i] = static_cast<_ExtentT>(__div_result.second); - } - __coords[_Rank - 1] = static_cast<_ExtentT>(__quotient % __extents_[_Rank - 1]); - return __coords; - } - } -}; - -//! @brief Lightweight device-side wrapper providing coordinate-indexed access to strided tensor data. -//! -//! Wraps a data pointer, per-dimension strides, and an accessor into a callable that maps -//! multi-dimensional coordinates to element references. -template -struct __partial_tensor -{ - _Tp* __ptr; - ::cuda::std::array<_StrideT, _Rank> __strides; - _Accessor __accessor; - - //! @brief Compute the linear offset for the given multi-dimensional coordinates. - //! - //! @param[in] __coords Array of per-dimension coordinates - //! @return Linear offset into the tensor storage - template - [[nodiscard]] _CCCL_DEVICE_API _StrideT __offset(const ::cuda::std::array<_CoordT, _Rank>& __coords) const noexcept - { - _StrideT __offset = 0; - _CCCL_PRAGMA_UNROLL_FULL() - for (int __i = 0; __i < int{_Rank}; ++__i) - { - __offset += static_cast<_StrideT>(__coords[__i]) * __strides[__i]; - } - return __offset; - } - - //! @brief Access the element at the given multi-dimensional coordinates. - //! - //! @param[in] __coords Array of per-dimension coordinates - //! @return Reference to the element at the computed offset - template - [[nodiscard]] _CCCL_DEVICE_API decltype(auto) - operator()(const ::cuda::std::array<_CoordT, _Rank>& __coords) const noexcept - { - return __accessor.access(const_cast<::cuda::std::remove_const_t<_Tp>*>(__ptr), __offset(__coords)); - } -}; -} // namespace cuda::experimental - -#include - -#endif // _CUDAX__COPY_TENSOR_ITERATOR_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy/vector_access.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy/vector_access.cuh deleted file mode 100644 index 28abcdb0..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy/vector_access.cuh +++ /dev/null @@ -1,74 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX__COPY_VECTOR_ACCESS_H -#define _CUDAX__COPY_VECTOR_ACCESS_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#if !_CCCL_COMPILER(NVRTC) -# include -# include -#endif // !_CCCL_COMPILER(NVRTC) - -#include - -#include - -namespace cuda::experimental -{ -//! @brief Aligned storage type for vectorized memory access of a given byte width. -template <::cuda::std::size_t _VectorBytes> -struct alignas(_VectorBytes) __vector_access -{ - char __data[_VectorBytes]; -}; - -// 32-byte accesses are supported since CTK 13.0 -#if _CCCL_CTK_AT_LEAST(13, 0) -inline constexpr auto __max_vector_access = 32; -#else -inline constexpr auto __max_vector_access = 16; -#endif // _CCCL_CTK_AT_LEAST(13, 0) - -#if !_CCCL_COMPILER(NVRTC) - -template <::cuda::std::size_t _VectorBytes> -using __vector_access_t = __vector_access<_VectorBytes>; - -//! @brief Query the maximum vector access width supported by the current GPU architecture. -//! -//! @return Maximum vector width in bytes (32 for SM >= 10.0, 16 otherwise) -[[nodiscard]] _CCCL_HOST_API inline ::cuda::std::size_t __max_gpu_arch_vector_size() noexcept -{ -# if _CCCL_CTK_AT_LEAST(13, 0) - const auto __dev_id = ::cuda::__driver::__cudevice_to_ordinal(::cuda::__driver::__ctxGetDevice()); - const auto __dev = ::cuda::devices[__dev_id]; - const auto __major = __dev.attribute<::cudaDevAttrComputeCapabilityMajor>(); - return (__major >= 10) ? 32 : 16; -# else // ^^^ _CCCL_CTK_AT_LEAST(13, 0) ^^^ / vvv _CCCL_CTK_BELOW(13, 0) vvv - return 16; -# endif // _CCCL_CTK_BELOW(13, 0) -} - -#endif // !_CCCL_COMPILER(NVRTC) -} // namespace cuda::experimental - -#include - -#endif // _CUDAX__COPY_VECTOR_ACCESS_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/abs_integer.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/abs_integer.cuh deleted file mode 100644 index 0c447b3f..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/abs_integer.cuh +++ /dev/null @@ -1,57 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef __CUDAX_COPY_ABS_INTEGER_H -#define __CUDAX_COPY_ABS_INTEGER_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#if !_CCCL_COMPILER(NVRTC) - -# include -# include -# include -# include - -# include - -namespace cuda::experimental -{ -//! @brief Returns the absolute value of an integer. Identity for unsigned types. -//! -//! @param[in] __value Integer value -//! @return Absolute value of @p __value -_CCCL_TEMPLATE(class _Tp) -_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _Tp __abs_integer(_Tp __value) noexcept -{ - if constexpr (::cuda::std::is_signed_v<_Tp>) - { - return ::cuda::std::abs(__value); - } - else - { - return __value; - } -} -} // namespace cuda::experimental - -# include - -#endif // !_CCCL_COMPILER(NVRTC) -#endif // __CUDAX_COPY_ABS_INTEGER_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/mdspan_d2h_h2d.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/mdspan_d2h_h2d.cuh deleted file mode 100644 index 0b189776..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/mdspan_d2h_h2d.cuh +++ /dev/null @@ -1,263 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef __CUDAX_COPY_MDSPAN_D2H_H2D_H -#define __CUDAX_COPY_MDSPAN_D2H_H2D_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#if !_CCCL_COMPILER(NVRTC) - -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include - -# include -# include -# include - -# include - -namespace cuda::experimental -{ -//! @brief Internal implementation of @ref copy_bytes for host/device mdspan transfers. -//! -//! Validates preconditions, converts mdspans to raw tensor descriptors, simplifies the paired layout -//! (sort, flip negative strides, coalesce), then dispatches a batched asynchronous memcpy. -//! -//! @param[in] __src Source mdspan -//! @param[out] __dst Destination mdspan -//! @param[in] __direction Copy direction (host-to-device or device-to-host) -//! @param[in] __stream CUDA stream for the asynchronous transfer -template -_CCCL_HOST_API void __copy_bytes_impl( - ::cuda::std::mdspan<_TpIn, _ExtentsIn, _LayoutPolicyIn, _AccessorPolicyIn> __src, - ::cuda::std::mdspan<_TpOut, _ExtentsOut, _LayoutPolicyOut, _AccessorPolicyOut> __dst, - [[maybe_unused]] __copy_direction __direction, - ::cuda::stream_ref __stream) -{ - namespace cudax = ::cuda::experimental; - static_assert(::cuda::std::is_same_v<::cuda::std::remove_cv_t<_TpIn>, ::cuda::std::remove_cv_t<_TpOut>>, - "cudax::copy_bytes: TpIn and TpOut must be the same type"); - static_assert(::cuda::is_trivially_copyable_v<_TpIn>, "TpIn must be trivially copyable"); - static_assert(!::cuda::std::is_const_v<_TpOut>, "TpOut must not be const"); - static_assert(::cuda::__is_cuda_mdspan_layout_v<_LayoutPolicyIn>, - "cudax::copy_bytes: LayoutPolicyIn must be a predefined layout policy"); - static_assert(::cuda::__is_cuda_mdspan_layout_v<_LayoutPolicyOut>, - "cudax::copy_bytes: LayoutPolicyOut must be a predefined layout policy"); - using __default_accessor_in = ::cuda::std::default_accessor<_TpIn>; - using __default_accessor_out = ::cuda::std::default_accessor<_TpOut>; - static_assert(::cuda::std::is_convertible_v<_AccessorPolicyIn, __default_accessor_in>, - "cudax::copy_bytes: AccessorPolicyIn must be convertible to cuda::std::default_accessor"); - static_assert(::cuda::std::is_convertible_v<_AccessorPolicyOut, __default_accessor_out>, - "cudax::copy_bytes: AccessorPolicyOut must be convertible to cuda::std::default_accessor"); - if (__stream.get() == nullptr) - { - _CCCL_THROW(::std::invalid_argument, "cudax::copy_bytes: stream must not be nullptr"); - } - if (__src.size() != __dst.size()) - { - _CCCL_THROW(::std::invalid_argument, "cudax::copy_bytes: mdspans must have the same size"); - } - - const auto __tensor_size = __src.size(); - if (__tensor_size == 0) - { - return; - } - if (__src.data_handle() == nullptr || __dst.data_handle() == nullptr) - { - _CCCL_THROW(::std::invalid_argument, "cudax::copy_bytes: mdspan data handle must not be nullptr"); - } - if (!::cuda::std::is_sufficiently_aligned(__src.data_handle())) - { - _CCCL_THROW(::std::invalid_argument, "cudax::copy_bytes: source mdspan must be sufficiently aligned"); - } - if (!::cuda::std::is_sufficiently_aligned(__dst.data_handle())) - { - _CCCL_THROW(::std::invalid_argument, "cudax::copy_bytes: destination mdspan must be sufficiently aligned"); - } - if (cudax::__has_interleaved_stride_order(__dst)) - { - _CCCL_THROW(::std::invalid_argument, - "cudax::copy_bytes: destination mdspan must not have interleaved stride order"); - } - - if (__tensor_size == 1) // rank == 0 also falls into this case - { - auto __src_ptr = __src.data_handle(); - auto __dst_ptr = __dst.data_handle(); - if constexpr (::cuda::__is_layout_stride_relaxed_v<_LayoutPolicyIn>) - { - __src_ptr += __src.mapping().offset(); - } - if constexpr (::cuda::__is_layout_stride_relaxed_v<_LayoutPolicyOut>) - { - __dst_ptr += __dst.mapping().offset(); - } - ::cuda::__driver::__memcpyAsync(__dst_ptr, __src_ptr, sizeof(_TpIn), __stream.get()); - return; - } - if constexpr (_ExtentsIn::rank() > 0 && _ExtentsOut::rank() > 0) - { - using __extent_t = ::cuda::std::common_type_t; - using __stride_t = - ::cuda::std::common_type_t, - cudax::__mdspan_stride_t<_LayoutPolicyOut, decltype(__dst.mapping())>>; - constexpr auto __max_rank = ::cuda::std::max(_ExtentsIn::rank(), _ExtentsOut::rank()); - const auto __src_raw = cudax::__to_raw_tensor<__extent_t, __stride_t, __max_rank>(__src); - const auto __dst_raw = cudax::__to_raw_tensor<__extent_t, __stride_t, __max_rank>(__dst); - if (!cudax::__same_extents(__src_raw, __dst_raw)) - { - _CCCL_THROW(::std::invalid_argument, - "cudax::copy_bytes: mdspans must have the same extents (after removing singleton dimensions)"); - } - - auto __src_simplified = __src_raw; - auto __dst_simplified = __dst_raw; - cudax::__sort_by_stride_paired(__src_simplified, __dst_simplified); - cudax::__flip_negative_strides_paired(__src_simplified, __dst_simplified); - cudax::__coalesce_paired(__src_simplified, __dst_simplified); - - const bool __both_stride1 = (__src_simplified.__strides[0] == 1) && (__dst_simplified.__strides[0] == 1); - const __extent_t __tile_size = __both_stride1 ? __src_simplified.__extents[0] : __extent_t{1}; - const auto __src_iter = (__tile_size > 1) ? __src_simplified : cudax::__reverse_modes(__src_raw); - const auto __dst_iter = (__tile_size > 1) ? __dst_simplified : cudax::__reverse_modes(__dst_raw); - - const auto __num_tiles = __tensor_size / __tile_size; - const auto __copy_bytes = __tile_size * sizeof(_TpIn); - _CCCL_ASSERT(__tensor_size % __tile_size == 0, "cudax::copy_bytes: tensor size must be divisible by tile size"); - __tile_iterator_linearized<__extent_t, __stride_t, _TpIn, __max_rank> __src_tiles_iterator(__src_iter, __tile_size); - __tile_iterator_linearized<__extent_t, __stride_t, _TpOut, __max_rank> __dst_tiles_iterator(__dst_iter, __tile_size); - - cudax::__memcpy_batch_tiles( - __src_tiles_iterator, - __dst_tiles_iterator, - __num_tiles, - __copy_bytes, - __direction, - __src.data_handle(), - __dst.data_handle(), - __stream); - } -} - -/*********************************************************************************************************************** - * Public API - **********************************************************************************************************************/ - -//! @rst -//! .. _cudax-copy-bytes: -//! -//! Asynchronous byte-wise mdspan copy -//! ------------------------------------ -//! -//! ``copy_bytes`` asynchronously copies elements between a host ``mdspan`` and a device ``mdspan`` on the given -//! CUDA stream. Two overloads are provided: host-to-device and device-to-host. -//! -//! - Source and destination must have the same total number of elements and identical extents -//! (after removing extent-1 dimensions). -//! - The implementation supports any stride value independently for source and destination mdspans. -//! - Element types must be trivially copyable and (ignoring cv-qualification) the same type. -//! - Layout policies must be one of the predefined ``cuda::std`` layout policies -//! (``layout_right``, ``layout_left``, ``layout_stride``) or ``cuda::layout_stride_relaxed``. -//! - Accessor policies must be convertible to ``cuda::std::default_accessor``. -//! - The destination must not have an interleaved stride order. -//! -//! The implementation is optimized to maximize the contiguous memory regions to copy and relies on batched asynchronous -//! memcpy. -//! -//! .. code-block:: c++ -//! -//! #include -//! -//! using extents_t = cuda::std::dims<2>; -//! cuda::host_mdspan src(src_ptr, extents); -//! cuda::device_mdspan dst(dst_ptr, extents); -//! cuda::experimental::copy_bytes(src, dst, stream); -//! -//! @endrst -//! @param[in] __src Source mdspan -//! @param[out] __dst Destination mdspan -//! @param[in] __stream CUDA stream for the asynchronous transfer -template -_CCCL_HOST_API void copy_bytes(::cuda::host_mdspan<_TpIn, _ExtentsIn, _LayoutPolicyIn, _AccessorPolicyIn> __src, - ::cuda::device_mdspan<_TpOut, _ExtentsOut, _LayoutPolicyOut, _AccessorPolicyOut> __dst, - ::cuda::stream_ref __stream) -{ - using __src_type = ::cuda::std::mdspan<_TpIn, _ExtentsIn, _LayoutPolicyIn, _AccessorPolicyIn>; - using __dst_type = ::cuda::std::mdspan<_TpOut, _ExtentsOut, _LayoutPolicyOut, _AccessorPolicyOut>; - ::cuda::experimental::__copy_bytes_impl( - static_cast<__src_type>(__src), static_cast<__dst_type>(__dst), __copy_direction::__host_to_device, __stream); -} - -//! @brief Asynchronously copies bytes from a device mdspan to a host mdspan. -//! -//! @param[in] __src Source device mdspan -//! @param[out] __dst Destination host mdspan -//! @param[in] __stream CUDA stream for the asynchronous transfer -template -_CCCL_HOST_API void copy_bytes(::cuda::device_mdspan<_TpIn, _ExtentsIn, _LayoutPolicyIn, _AccessorPolicyIn> __src, - ::cuda::host_mdspan<_TpOut, _ExtentsOut, _LayoutPolicyOut, _AccessorPolicyOut> __dst, - ::cuda::stream_ref __stream) -{ - using __src_type = ::cuda::std::mdspan<_TpIn, _ExtentsIn, _LayoutPolicyIn, _AccessorPolicyIn>; - using __dst_type = ::cuda::std::mdspan<_TpOut, _ExtentsOut, _LayoutPolicyOut, _AccessorPolicyOut>; - ::cuda::experimental::__copy_bytes_impl( - static_cast<__src_type>(__src), static_cast<__dst_type>(__dst), __copy_direction::__device_to_host, __stream); -} -} // namespace cuda::experimental - -# include - -#endif // !_CCCL_COMPILER(NVRTC) -#endif // __CUDAX_COPY_MDSPAN_D2H_H2D_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/mdspan_to_raw_tensor.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/mdspan_to_raw_tensor.cuh deleted file mode 100644 index 36e1e475..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/mdspan_to_raw_tensor.cuh +++ /dev/null @@ -1,134 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef __CUDAX_COPY_MDSPAN_TO_RAW_TENSOR_H -#define __CUDAX_COPY_MDSPAN_TO_RAW_TENSOR_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#if !_CCCL_COMPILER(NVRTC) - -# include -# include -# include -# include - -# include - -# include - -namespace cuda::experimental -{ -//! @brief Extracts the stride type from a layout mapping, defaulting to `index_type` when absent. -template -struct __mapping_stride_type -{ - using __type = typename _Mapping::index_type; -}; - -template -struct __mapping_stride_type<::cuda::layout_stride_relaxed, _Mapping> -{ - using __type = typename _Mapping::offset_type; -}; - -//! @brief Convenience alias: stride type of a layout mapping for given extents and layout policy. -//! -//! For `layout_stride_relaxed`, uses `offset_type` (signed) since strides can be negative. -//! For other layouts, uses `stride_type` if available, otherwise `index_type`. -template -using __mdspan_stride_t = typename __mapping_stride_type<_LayoutPolicy, ::cuda::std::remove_cvref_t<_Mapping>>::__type; - -//! @brief Convenience alias: `__raw_tensor` type produced by @ref __to_raw_tensor for a given mdspan. -template > -using __to_raw_tensor_t = - __raw_tensor, - typename _Mdspan::element_type, - _Mdspan::rank()>; - -//! @brief Converts an mdspan to a @ref __raw_tensor with explicitly specified extent, stride types. -//! -//! Extent-1 modes are removed from the resulting tensor. -//! -//! @param[in] __mdspan Source mdspan view -//! @return @ref __raw_tensor descriptor with data pointer, rank, extents, and strides -template -[[nodiscard]] -_CCCL_HOST_API constexpr __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank> -__to_raw_tensor(const ::cuda::std::mdspan<_Tp, _Extents, _LayoutPolicy, _AccessorPolicy>& __mdspan) noexcept -{ - static_assert(_MaxRank >= _Extents::rank(), "_MaxRank must be at least _Extents::rank()"); - using __raw_tensor_t = __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>; - using __rank_t = typename _Extents::rank_type; - auto* __data = __mdspan.data_handle(); - if constexpr (::cuda::__is_layout_stride_relaxed_v<_LayoutPolicy>) - { - __data += __mdspan.mapping().offset(); - } - __raw_tensor_t __result{__data, 0, {}, {}}; - if constexpr (_Extents::rank() > 0) - { - __rank_t __r = 0; - for (__rank_t __i = 0; __i < _Extents::rank(); ++__i) - { - const auto __extent = static_cast<_ExtentT>(__mdspan.extent(__i)); - if (__extent != _ExtentT{1}) - { - __result.__extents[__r] = __extent; - __result.__strides[__r] = static_cast<_StrideT>(__mdspan.stride(__i)); - ++__r; - } - } - for (__rank_t __i = __r; __i < _MaxRank; ++__i) - { - __result.__extents[__i] = _ExtentT{1}; - } - __result.__rank = __r; - } - return __result; -} - -//! @brief Converts an mdspan to a @ref __raw_tensor using its native extent and stride types. -//! -//! Extent-1 modes are removed from the resulting tensor. -//! -//! @param[in] __mdspan Source mdspan view -//! @return @ref __raw_tensor descriptor with data pointer, rank, extents, and strides -template -[[nodiscard]] -_CCCL_HOST_API constexpr auto -__to_raw_tensor(const ::cuda::std::mdspan<_Tp, _Extents, _LayoutPolicy, _AccessorPolicy>& __mdspan) noexcept - -> __to_raw_tensor_t -{ - using __extent_t = typename _Extents::index_type; - using __stride_t = __mdspan_stride_t<_LayoutPolicy, decltype(__mdspan.mapping())>; - return ::cuda::experimental::__to_raw_tensor<__extent_t, __stride_t, _Extents::rank()>(__mdspan); -} -} // namespace cuda::experimental - -# include - -#endif // !_CCCL_COMPILER(NVRTC) -#endif // __CUDAX_COPY_MDSPAN_TO_RAW_TENSOR_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/memcpy_batch_tiles.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/memcpy_batch_tiles.cuh deleted file mode 100644 index 70c91a8c..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/memcpy_batch_tiles.cuh +++ /dev/null @@ -1,205 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef __CUDAX_COPY_MEMCPY_BATCH_TILES_H -#define __CUDAX_COPY_MEMCPY_BATCH_TILES_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#if !_CCCL_COMPILER(NVRTC) - -# include -# include -# include -# include -# include -# include - -# include - -# include - -# include - -namespace cuda::experimental -{ -//! @brief Iterator that maps a linear tile index to a pointer into a strided raw tensor. -template -struct __tile_iterator_linearized -{ - const __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank> __tensor_; - ::cuda::std::array<_ExtentT, _MaxRank> __extent_products_; - const _ExtentT __contiguous_size_; - - //! @brief Constructs the iterator from a raw tensor and contiguous tile size. - //! - //! @param[in] __tensor Raw tensor descriptor - //! @param[in] __contiguous_size Number of contiguous elements per tile - _CCCL_HOST_API explicit __tile_iterator_linearized(const __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>& __tensor, - _ExtentT __contiguous_size) noexcept - : __tensor_{__tensor} - , __extent_products_{} - , __contiguous_size_{__contiguous_size} - { - // Precomputes exclusive prefix products of extents so that each `operator()` call decomposes a flat index into - // multi-dimensional coordinates and computes the corresponding byte offset. - ::cuda::std::exclusive_scan( - __tensor.__extents.data(), - __tensor.__extents.data() + __tensor.__rank, - __extent_products_.data(), - _ExtentT{1}, - ::cuda::std::multiplies<>{}); - } - - //! @brief Returns a pointer to the first element of the tile at @p __tile_idx. - //! - //! @param[in] __tile_idx linear tile index - //! @return Pointer into the tensor at the computed multi-dimensional offset - [[nodiscard]] _CCCL_HOST_API _Tp* operator()(_ExtentT __tile_idx) const noexcept - { - using __uextent_t = ::cuda::std::make_unsigned_t<_ExtentT>; - const auto __index = __tile_idx * __contiguous_size_; - const auto& __extents = __tensor_.__extents; - const auto& __strides = __tensor_.__strides; - if (__tensor_.__rank == 1) - { - return __tensor_.__data + __index * __strides[0]; - } - const auto __extent0 = static_cast<__uextent_t>(__extents[0]); - _StrideT __offset = (__index % __extent0) * __strides[0]; // __extent_products_[0] == 1 - for (::cuda::std::size_t __i = 1; __i < __tensor_.__rank; ++__i) - { - const auto __extent_product = static_cast<__uextent_t>(__extent_products_[__i]); - const auto __coord = static_cast<_StrideT>((__index / __extent_product) % __extents[__i]); - __offset += __coord * __strides[__i]; - } - return __tensor_.__data + __offset; - } -}; - -# if _CCCL_CTK_AT_LEAST(13, 0) - -//! @brief Builds the `CUmemcpyAttributes` descriptor for a batch async memcpy. -//! -//! @param[in] __direction Copy direction (host-to-device or device-to-host) -//! @param[in] __src_ptr Source pointer (used to query device ordinal for D2H) -//! @param[in] __dst_ptr Destination pointer (used to query device ordinal for H2D) -//! @return Populated `CUmemcpyAttributes` struct -[[nodiscard]] _CCCL_HOST_API inline ::CUmemcpyAttributes -__get_memcpy_attributes(__copy_direction __direction, const void* __src_ptr, const void* __dst_ptr) noexcept -{ - if (__direction == __copy_direction::__host_to_device) - { - const int __device_ordinal = - ::cuda::__driver::__pointerGetAttribute<::CU_POINTER_ATTRIBUTE_DEVICE_ORDINAL>(__dst_ptr); - return ::CUmemcpyAttributes{ - ::CU_MEMCPY_SRC_ACCESS_ORDER_ANY, - ::CUmemLocation{::CU_MEM_LOCATION_TYPE_HOST, 0}, - ::CUmemLocation{::CU_MEM_LOCATION_TYPE_DEVICE, __device_ordinal}, - 0}; - } - const int __device_ordinal = - ::cuda::__driver::__pointerGetAttribute<::CU_POINTER_ATTRIBUTE_DEVICE_ORDINAL>(__src_ptr); - return ::CUmemcpyAttributes{ - ::CU_MEMCPY_SRC_ACCESS_ORDER_ANY, - ::CUmemLocation{::CU_MEM_LOCATION_TYPE_DEVICE, __device_ordinal}, - ::CUmemLocation{::CU_MEM_LOCATION_TYPE_HOST, 0}, - 0}; -} - -# endif // _CCCL_CTK_AT_LEAST(13, 0) - -//! @brief Submits an asynchronous batch memcpy for every tile. -//! -//! - Uses `cuMemcpyBatchAsync` on CTK 13.0+ with stack-allocated arrays for small tile counts, -//! falling back to heap allocation when the count exceeds a fixed threshold. -//! - On older toolkits, issues individual `cuMemcpyAsync` calls per tile. -//! -//! @param[in] __src_tiles_iterator Tile iterator for the source tensor -//! @param[in] __dst_tiles_iterator Tile iterator for the destination tensor -//! @param[in] __num_tiles Number of tiles to copy -//! @param[in] __copy_size_bytes Byte size of each tile -//! @param[in] __direction Copy direction -//! @param[in] __src_data_handle Source base pointer (for attribute query) -//! @param[in] __dst_data_handle Destination base pointer (for attribute query) -//! @param[in] __stream CUDA stream -template -_CCCL_HOST_API inline void __memcpy_batch_tiles( - const _SrcTileIterator& __src_tiles_iterator, - const _DstTileIterator& __dst_tiles_iterator, - ::cuda::std::size_t __num_tiles, - ::cuda::std::size_t __copy_size_bytes, - [[maybe_unused]] __copy_direction __direction, - [[maybe_unused]] const void* __src_data_handle, - [[maybe_unused]] void* __dst_data_handle, - ::cuda::stream_ref __stream) -{ - using ::cuda::std::size_t; -# if _CCCL_CTK_AT_LEAST(13, 0) - auto __attributes = ::cuda::experimental::__get_memcpy_attributes(__direction, __src_data_handle, __dst_data_handle); - const auto __memcpy_batch_async_lambda = [&](auto __src_ptrs, auto __dst_ptrs, auto __sizes) { - for (size_t __tile_idx = 0; __tile_idx < __num_tiles; ++__tile_idx) - { - __src_ptrs[__tile_idx] = __src_tiles_iterator(__tile_idx); - __dst_ptrs[__tile_idx] = __dst_tiles_iterator(__tile_idx); - __sizes[__tile_idx] = __copy_size_bytes; - } - size_t __attribute_indices = 0; - ::cuda::__driver::__memcpyBatchAsync( - __dst_ptrs, - __src_ptrs, - __sizes, - __num_tiles, - &__attributes, - &__attribute_indices, - /*num_attributes=*/1, - __stream.get()); - }; - constexpr size_t __max_tiles = 16; - if (__num_tiles > __max_tiles) - { - auto __src_ptrs = new const void*[__num_tiles]; - auto __dst_ptrs = new void*[__num_tiles]; - auto __sizes = new size_t[__num_tiles]; - __memcpy_batch_async_lambda(__src_ptrs, __dst_ptrs, __sizes); - delete[] __src_ptrs; - delete[] __dst_ptrs; - delete[] __sizes; - } - else - { - ::cuda::std::array __src_ptr_array{}; - ::cuda::std::array __dst_ptr_array{}; - ::cuda::std::array __sizes{}; - __memcpy_batch_async_lambda(__src_ptr_array.data(), __dst_ptr_array.data(), __sizes.data()); - } -# else - for (size_t __tile_idx = 0; __tile_idx < __num_tiles; ++__tile_idx) - { - const auto __src_ptr = static_cast(__src_tiles_iterator(__tile_idx)); - const auto __dst_ptr = static_cast(__dst_tiles_iterator(__tile_idx)); - ::cuda::__driver::__memcpyAsync(__dst_ptr, __src_ptr, __copy_size_bytes, __stream.get()); - } -# endif // _CCCL_CTK_AT_LEAST(13, 0) -} -} // namespace cuda::experimental - -# include - -#endif // !_CCCL_COMPILER(NVRTC) -#endif // __CUDAX_COPY_MEMCPY_BATCH_TILES_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/print_raw_tensor.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/print_raw_tensor.cuh deleted file mode 100644 index adf4a537..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/print_raw_tensor.cuh +++ /dev/null @@ -1,68 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef __CUDAX_COPY_PRINT_RAW_TENSOR_H -#define __CUDAX_COPY_PRINT_RAW_TENSOR_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#if !_CCCL_COMPILER(NVRTC) - -# include - -# include - -# include - -# include - -namespace cuda::experimental -{ -//! @brief Prints a raw tensor's extents and strides to stdout in the format `(extents):(strides)`. -//! -//! @param[in] __tensor Raw tensor to print -template -_CCCL_HOST_API void __println(const __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>& __tensor) -{ - const auto __rank = static_cast(__tensor.__rank); - ::printf("("); - for (int __i = 0; __i < __rank - 1; ++__i) - { - ::printf("%llu, ", static_cast(__tensor.__extents[__i])); - } - if (__rank > 0) - { - ::printf("%llu", static_cast(__tensor.__extents[__rank - 1])); - } - ::printf("):("); - for (int __i = 0; __i < __rank - 1; ++__i) - { - ::printf("%lld, ", static_cast(__tensor.__strides[__i])); - } - if (__rank > 0) - { - ::printf("%lld", static_cast(__tensor.__strides[__rank - 1])); - } - ::printf(")\n"); -} -} // namespace cuda::experimental - -# include - -#endif // !_CCCL_COMPILER(NVRTC) -#endif // __CUDAX_COPY_PRINT_RAW_TENSOR_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/simplify_paired.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/simplify_paired.cuh deleted file mode 100644 index eb0287a0..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/simplify_paired.cuh +++ /dev/null @@ -1,212 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef __CUDAX_COPY_SIMPLIFY_PAIRED_H -#define __CUDAX_COPY_SIMPLIFY_PAIRED_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#if !_CCCL_COMPILER(NVRTC) - -# include -# include -# include -# include - -# include - -# include - -namespace cuda::experimental -{ -//! @brief Reverses the order of active modes in a raw tensor. -//! -//! This helps to get a single logic for __tile_iterator_linearized -//! -//! @param[in] __input Raw tensor whose modes are reversed -//! @return New raw tensor with extents and strides in reversed mode order -template -[[nodiscard]] _CCCL_HOST_API __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank> -__reverse_modes(const __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>& __input) noexcept -{ - using __raw_tensor_t = __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>; - using __rank_t = typename __raw_tensor_t::__rank_t; - __raw_tensor_t __result{__input.__data, __input.__rank, {}, {}}; - _CCCL_ASSERT(__input.__rank > 0, "cudax::reverse_modes: input tensor must have rank > 0"); - for (__rank_t __i = 0; __i < __input.__rank; ++__i) - { - const auto __j = __input.__rank - 1 - __i; - __result.__extents[__i] = __input.__extents[__j]; - __result.__strides[__i] = __input.__strides[__j]; - } - return __result; -} - -// lambdas are painful without --extended-lambda and when used with __host__ __device__ functions -struct __mode_compare_paired -{ - template - [[nodiscard]] _CCCL_HOST_DEVICE_API bool - operator()(const ::cuda::std::tuple<_ExtentT, _SrcStrideT, _DstStrideT>& __lhs, - const ::cuda::std::tuple<_ExtentT, _SrcStrideT, _DstStrideT>& __rhs) const noexcept - { - namespace cudax = ::cuda::experimental; - const auto __src_lhs = ::cuda::std::get<1>(__lhs); - const auto __src_rhs = ::cuda::std::get<1>(__rhs); - const auto __dst_lhs = ::cuda::std::get<2>(__lhs); - const auto __dst_rhs = ::cuda::std::get<2>(__rhs); - return cudax::__abs_integer(__src_lhs) < cudax::__abs_integer(__src_rhs) - || (cudax::__abs_integer(__src_lhs) == cudax::__abs_integer(__src_rhs) - && cudax::__abs_integer(__dst_lhs) < cudax::__abs_integer(__dst_rhs)); - } -}; - -//! @brief Sorts a source/destination tensor pair by ascending absolute destination stride. -//! -//! Both tensors are reordered in lockstep so that corresponding modes remain paired. -//! -//! @pre @p __src and @p __dst must have the same extents -//! -//! @param[in,out] __src Source raw tensor (modes reordered in place) -//! @param[in,out] __dst Destination raw tensor (modes reordered in place) -template -_CCCL_HOST_API void __sort_by_stride_paired(__raw_tensor<_ExtentT, _SrcStrideT, _TpSrc, _MaxRank>& __src, - __raw_tensor<_ExtentT, _DstStrideT, _TpDst, _MaxRank>& __dst) noexcept -{ - namespace cudax = ::cuda::experimental; - using __raw_tensor_t = __raw_tensor<_ExtentT, _SrcStrideT, _TpSrc, _MaxRank>; - using __rank_t = typename __raw_tensor_t::__rank_t; - using __mode_t = ::cuda::std::tuple<_ExtentT, _SrcStrideT, _DstStrideT>; - const auto __rank = __src.__rank; - _CCCL_ASSERT(cudax::__same_extents(__src, __dst), "Source and destination tensors must have the same extents"); - ::cuda::std::array<__mode_t, _MaxRank> __modes{}; - for (__rank_t __i = 0; __i < __rank; ++__i) - { - __modes[__i] = {__src.__extents[__i], __src.__strides[__i], __dst.__strides[__i]}; - } - ::cuda::std::stable_sort(__modes.begin(), __modes.begin() + __rank, __mode_compare_paired{}); - for (__rank_t __i = 0; __i < __rank; ++__i) - { - ::cuda::std::tie(__src.__extents[__i], __src.__strides[__i], __dst.__strides[__i]) = __modes[__i]; - } - __dst.__extents = __src.__extents; -} - -//! @brief Flips modes where both source and destination strides are negative. -//! -//! For each such mode, the base pointer is advanced to the last element and the stride is negated, yielding an -//! equivalent tensor with positive strides. -//! -//! @pre @p __src and @p __dst must have the same extents -//! -//! @param[in,out] __src Source raw tensor (data pointer and strides may be modified) -//! @param[in,out] __dst Destination raw tensor (data pointer and strides may be modified) -template -_CCCL_HOST_API void __flip_negative_strides_paired( - [[maybe_unused]] __raw_tensor<_ExtentT, _SrcStrideT, _TpSrc, _MaxRank>& __src, - [[maybe_unused]] __raw_tensor<_ExtentT, _DstStrideT, _TpDst, _MaxRank>& __dst) noexcept -{ - if constexpr (::cuda::std::is_signed_v<_SrcStrideT> && ::cuda::std::is_signed_v<_DstStrideT>) - { - using __raw_tensor_t = __raw_tensor<_ExtentT, _SrcStrideT, _TpSrc, _MaxRank>; - using __rank_t = typename __raw_tensor_t::__rank_t; - _CCCL_ASSERT(::cuda::experimental::__same_extents(__src, __dst), - "cudax::flip_negative_strides_paired: Source and destination tensors must have the same extents"); - for (__rank_t __i = 0; __i < __src.__rank; ++__i) - { - if (__src.__strides[__i] < 0 && __dst.__strides[__i] < 0) - { - const auto __extent = __src.__extents[__i]; - const auto __src_adjustment = static_cast<_SrcStrideT>(__extent - 1) * __src.__strides[__i]; - const auto __dst_adjustment = static_cast<_DstStrideT>(__extent - 1) * __dst.__strides[__i]; - __src.__data += __src_adjustment; - __dst.__data += __dst_adjustment; - __src.__strides[__i] = -__src.__strides[__i]; - __dst.__strides[__i] = -__dst.__strides[__i]; - } - } - } -} - -//! @brief Merges adjacent modes that are contiguous in both source and destination tensors. -//! -//! Two consecutive modes are merged when `extent[i-1] * stride[i-1] == stride[i]` holds for both tensors. -//! The resulting tensor pair has fewer modes but represents the same memory layout. -//! -//! @pre @p __src and @p __dst must have the same extents -//! -//! @param[in,out] __src Source raw tensor (rank and modes may be reduced) -//! @param[in,out] __dst Destination raw tensor (rank and modes may be reduced) -template -_CCCL_HOST_API void __coalesce_paired(__raw_tensor<_ExtentT, _SrcStrideT, _TpSrc, _MaxRank>& __src, - __raw_tensor<_ExtentT, _DstStrideT, _TpDst, _MaxRank>& __dst) noexcept -{ - _CCCL_ASSERT(::cuda::experimental::__same_extents(__src, __dst), - "Source and destination tensors must have the same extents"); - if (__src.__rank <= 1) - { - return; - } - using __raw_tensor_t = __raw_tensor<_ExtentT, _SrcStrideT, _TpSrc, _MaxRank>; - using __rank_t = typename __raw_tensor_t::__rank_t; - __rank_t __out_r = 1; - for (__rank_t __i = 1; __i < __src.__rank; ++__i) - { - const auto __src_prev_extent = static_cast<_SrcStrideT>(__src.__extents[__out_r - 1]); - const auto __dst_prev_extent = static_cast<_DstStrideT>(__src.__extents[__out_r - 1]); - const bool __src_contiguous = (__src_prev_extent * __src.__strides[__out_r - 1] == __src.__strides[__i]); - const bool __dst_contiguous = (__dst_prev_extent * __dst.__strides[__out_r - 1] == __dst.__strides[__i]); - if (__src_contiguous && __dst_contiguous) - { - __src.__extents[__out_r - 1] *= __src.__extents[__i]; - continue; - } - __src.__extents[__out_r] = __src.__extents[__i]; - __src.__strides[__out_r] = __src.__strides[__i]; - __dst.__strides[__out_r] = __dst.__strides[__i]; - ++__out_r; - } - for (__rank_t __i = __out_r; __i < _MaxRank; ++__i) - { - __src.__extents[__i] = _ExtentT{1}; - } - __src.__rank = __out_r; - __dst.__rank = __out_r; - __dst.__extents = __src.__extents; -} -} // namespace cuda::experimental - -# include - -#endif // !_CCCL_COMPILER(NVRTC) -#endif // __CUDAX_COPY_SIMPLIFY_PAIRED_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/tensor_query.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/tensor_query.cuh deleted file mode 100644 index 689edb02..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/tensor_query.cuh +++ /dev/null @@ -1,180 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef __CUDAX_COPY_TENSOR_QUERY_H -#define __CUDAX_COPY_TENSOR_QUERY_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#if !_CCCL_COMPILER(NVRTC) - -# include -# include -# include -# include - -# include -# include -# include - -# include - -namespace cuda::experimental -{ -//! @brief Checks whether two raw tensors have the same rank and identical extents. -//! -//! @param[in] __tensor_in First raw tensor -//! @param[in] __tensor_out Second raw tensor -//! @return true if rank and all extents match element-wise -template -[[nodiscard]] _CCCL_HOST_API constexpr bool -__same_extents(const __raw_tensor<_ExtentTIn, _StrideTIn, _TpIn, _MaxRankIn>& __tensor_in, - const __raw_tensor<_ExtentTOut, _StrideTOut, _TpOut, _MaxRankOut>& __tensor_out) noexcept -{ - if (__tensor_in.__rank != __tensor_out.__rank) - { - return false; - } - using __raw_tensor_t = __raw_tensor<_ExtentTIn, _StrideTIn, _TpIn, _MaxRankIn>; - using __rank_t = typename __raw_tensor_t::__rank_t; - for (__rank_t __i = 0; __i < __tensor_in.__rank; ++__i) - { - if (__tensor_in.__extents[__i] != __tensor_out.__extents[__i]) - { - return false; - } - } - return true; -} - -// lambdas are painful without --extended-lambda and when used with __host__ __device__ functions -template -struct __stride_compare -{ - const ::cuda::std::array<_StrideT, _MaxRank>& __strides; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(const _Idx __lhs, const _Idx __rhs) const noexcept - { - return ::cuda::experimental::__abs_integer(__strides[__lhs]) - < ::cuda::experimental::__abs_integer(__strides[__rhs]); - } -}; - -//! @brief Computes the mode permutation that orders a tensor by ascending absolute stride. -//! -//! @param[in] __tensor Raw tensor whose stride order is inspected -//! @return Mode permutation sorted by ascending absolute stride -template -[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::std::array<::cuda::std::size_t, _MaxRank> -__stride_order(const __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>& __tensor) noexcept -{ - ::cuda::std::array<::cuda::std::size_t, _MaxRank> __perm{}; - for (::cuda::std::size_t __i = 0; __i < _MaxRank; ++__i) - { - __perm[__i] = __i; - } - ::cuda::std::stable_sort( - __perm.begin(), __perm.begin() + __tensor.__rank, __stride_compare<_StrideT, _MaxRank>{__tensor.__strides}); - return __perm; -} - -//! @brief Reorders tensor modes by ascending absolute stride. -//! -//! After sorting, mode 0 has the smallest absolute stride (innermost) and mode rank-1 has the largest (outermost). -//! -//! @param[in] __tensor Raw tensor to sort -//! @return Raw tensor with modes reordered by ascending absolute stride -template -[[nodiscard]] _CCCL_HOST_API constexpr __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank> -__sort_by_stride(const __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>& __tensor) noexcept -{ - using __raw_tensor_t = __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank>; - using __rank_t = typename __raw_tensor_t::__rank_t; - const auto __rank = __tensor.__rank; - const auto __perm = ::cuda::experimental::__stride_order(__tensor); - - __raw_tensor<_ExtentT, _StrideT, _Tp, _MaxRank> __result{__tensor.__data, __rank}; - for (__rank_t __i = 0; __i < __rank; ++__i) - { - __result.__extents[__i] = __tensor.__extents[__perm[__i]]; - __result.__strides[__i] = __tensor.__strides[__perm[__i]]; - } - return __result; -} - -//! @brief Conservative check for interleaved stride order in tensor layouts. -//! -//! Sorts modes by ascending absolute stride, then verifies two conditions: -//! 1. No mode with extent > 1 has stride == 0 (broadcast) -//! 2. No mode's span (extent * |stride|) exceeds the next mode's |stride| -//! -//! Returns true when the layout fails this non-interleaving rule. This is stronger than a mathematical injectivity -//! check and may reject some layouts with distinct offsets. -//! -//! @param[in] __mdspan Mdspan view to inspect -//! @return true if the layout has interleaved strides -template -[[nodiscard]] _CCCL_HOST_API constexpr bool __has_interleaved_stride_order( - const ::cuda::std::mdspan<_Tp, _Extents, _LayoutPolicy, _AccessorPolicy>& __mdspan) noexcept -{ - if constexpr (_Extents::rank() > 0) - { - namespace cudax = ::cuda::experimental; - const auto __tensor = cudax::__to_raw_tensor(__mdspan); - const auto __sorted = cudax::__sort_by_stride(__tensor); - using __stride_t = decltype(__sorted.__strides[0]); - using __rank_t = typename _Extents::rank_type; - const auto& __extents = __sorted.__extents; - const auto& __strides = __sorted.__strides; - const auto __rank = __sorted.__rank; - for (__rank_t __i = 0; __i < __rank; ++__i) - { - if (__extents[__i] > 1 && __strides[__i] == 0) - { - return true; - } - } - for (__rank_t __i = 0; __i + 1 < __rank; ++__i) - { - const auto __extent = static_cast<__stride_t>(__extents[__i]); - if (__extent * cudax::__abs_integer(__strides[__i]) > cudax::__abs_integer(__strides[__i + 1])) - { - return true; - } - } - return false; - } - else - { - return false; - } -} -} // namespace cuda::experimental - -# include - -#endif // !_CCCL_COMPILER(NVRTC) -#endif // __CUDAX_COPY_TENSOR_QUERY_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/types.cuh b/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/types.cuh deleted file mode 100644 index df7efa6c..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__copy_bytes/types.cuh +++ /dev/null @@ -1,56 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef __CUDAX_COPY_TYPES_H -#define __CUDAX_COPY_TYPES_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#if !_CCCL_COMPILER(NVRTC) - -# include -# include - -# include - -namespace cuda::experimental -{ -//! @brief Raw tensor descriptor with dynamic rank, extents, and strides. -template -struct __raw_tensor -{ - using __rank_t = ::cuda::std::size_t; - - _Tp* __data; - __rank_t __rank; - ::cuda::std::array<_ExtentT, _MaxRank> __extents; - ::cuda::std::array<_StrideT, _MaxRank> __strides; -}; - -//! @brief Direction of an asynchronous memcpy operation. -enum class __copy_direction -{ - __host_to_device, - __device_to_host, -}; -} // namespace cuda::experimental - -# include - -#endif // !_CCCL_COMPILER(NVRTC) -#endif // __CUDAX_COPY_TYPES_H diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/capacity.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/capacity.cuh deleted file mode 100644 index e815384c..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/capacity.cuh +++ /dev/null @@ -1,126 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_CAPACITY_CUH -#define _CUDAX___CUCO_CAPACITY_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -namespace cuda::experimental::cuco -{ -//! @brief Rounds a requested capacity up to the smallest valid capacity for the given probing scheme -//! and bucket size. -//! -//! The probe stride is `_ProbingScheme::cg_size * _BucketSize`. For linear probing the result is a -//! multiple of the stride; for double hashing the probe cycle count `capacity / stride` is -//! additionally prime. The function is idempotent: applying it to an already valid capacity returns -//! the same value. -//! -//! @tparam _ProbingScheme Probing scheme type -//! @tparam _BucketSize Number of slots per bucket -//! @tparam _SizeType Size type -//! -//! @param __requested Requested capacity -//! -//! @return The smallest valid capacity that is greater than or equal to `__requested` -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _SizeType make_valid_capacity(_SizeType __requested) -{ - constexpr auto __stride = _SizeType{_ProbingScheme::cg_size * _BucketSize}; - const auto __cycles = ::cuda::ceil_div(::cuda::std::max(__requested, _SizeType{1}), __stride); - _SizeType __capacity{}; - if constexpr (is_double_hashing_v<_ProbingScheme>) - { - const auto __prime = detail::__next_prime(static_cast<::cuda::std::uint64_t>(__cycles)); - if (::cuda::mul_overflow(__capacity, __prime, __stride)) - { - _CCCL_THROW(::std::logic_error, "Invalid input capacity"); - } - } - else - { - const auto __num_buckets = __cycles + _SizeType{__requested == 0}; - if (::cuda::mul_overflow(__capacity, __num_buckets, __stride)) - { - _CCCL_THROW(::std::logic_error, "Invalid input capacity"); - } - } - return __capacity; -} - -//! @brief Rounds a requested capacity up to a valid capacity for a desired load factor. -//! -//! @tparam _ProbingScheme Probing scheme type -//! @tparam _BucketSize Number of slots per bucket -//! @tparam _SizeType Size type -//! -//! @param __requested Requested element count -//! @param __load_factor Desired load factor in (0, 1] -//! -//! @return The smallest valid capacity that fits `__requested` elements at `__load_factor` -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _SizeType make_valid_capacity(_SizeType __requested, double __load_factor) -{ - if (__load_factor <= 0. || !::cuda::in_range(__load_factor, 0., 1.)) - { - _CCCL_THROW(::std::logic_error, "Desired load factor must be in the range (0, 1]"); - } - const auto __scaled = ::cuda::std::ceil(static_cast(__requested) / __load_factor); - if (__scaled > static_cast(::cuda::std::numeric_limits<_SizeType>::max())) - { - _CCCL_THROW(::std::logic_error, - "Invalid load factor: requested capacity divided by load factor exceeds the maximum representable " - "value"); - } - return make_valid_capacity<_ProbingScheme, _BucketSize>(static_cast<_SizeType>(__scaled)); -} - -//! @brief Returns whether `__capacity` is already a valid capacity for the given probing scheme and -//! bucket size. -//! -//! @tparam _ProbingScheme Probing scheme type -//! @tparam _BucketSize Number of slots per bucket -//! @tparam _SizeType Size type -//! -//! @param __capacity Capacity to test -//! -//! @return `true` if `__capacity` needs no rounding -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool is_valid_capacity(_SizeType __capacity) -{ - return make_valid_capacity<_ProbingScheme, _BucketSize>(__capacity) == __capacity; -} -} // namespace cuda::experimental::cuco - -#include - -#endif // _CUDAX___CUCO_CAPACITY_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/bitwise_compare.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/bitwise_compare.cuh deleted file mode 100644 index 104049a2..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/bitwise_compare.cuh +++ /dev/null @@ -1,59 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_BITWISE_COMPARE_CUH -#define _CUDAX___CUCO_DETAIL_BITWISE_COMPARE_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include - -#include - -namespace cuda::experimental::cuco::detail -{ -//! @brief Bitwise equality comparison. -//! -//! @tparam _Tp Value type -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool __bitwise_compare(const _Tp& __lhs, const _Tp& __rhs) -{ - static_assert(::cuda::is_bitwise_comparable_v<_Tp>, - "Bitwise compared objects must have unique object representations or be explicitly declared safe."); - if constexpr (sizeof(_Tp) <= sizeof(::cuda::std::uint64_t) - || (sizeof(_Tp) == 2 * sizeof(::cuda::std::uint64_t) && _CCCL_HAS_INT128())) - { - using _Up = ::cuda::std::__make_nbit_uint_t::digits>; - return ::cuda::std::bit_cast<_Up>(__lhs) == ::cuda::std::bit_cast<_Up>(__rhs); - } - else - { - using _Array = ::cuda::std::array<::cuda::std::uint64_t, sizeof(_Tp) / sizeof(::cuda::std::uint64_t)>; - return ::cuda::std::bit_cast<_Array>(__lhs) == ::cuda::std::bit_cast<_Array>(__rhs); - } -} -} // namespace cuda::experimental::cuco::detail - -#include - -#endif // _CUDAX___CUCO_DETAIL_BITWISE_COMPARE_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/equal_wrapper.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/equal_wrapper.cuh deleted file mode 100644 index 0b180902..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/equal_wrapper.cuh +++ /dev/null @@ -1,131 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_EQUAL_WRAPPER_CUH -#define _CUDAX___CUCO_DETAIL_EQUAL_WRAPPER_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include - -#include - -#include - -namespace cuda::experimental::cuco::detail -{ -//! @brief Enum of equality comparison results. -enum class __equal_result : ::cuda::std::int8_t -{ - __unequal, - __equal, - __empty, - __available, -}; - -//! @brief Enum indicating whether the operation is an insert. -enum class __is_insert : ::cuda::std::int8_t -{ - __yes, - __no -}; - -//! @brief Key equality wrapper. -//! -//! @tparam _Tp Right-hand side element type -//! @tparam _Equal Equality callable -//! @tparam _AllowsDuplicates Duplicate key flag -template -struct __equal_wrapper -{ - _Tp __empty_sentinel; - _Tp __erased_sentinel; - _Equal __equal; - - //! @brief Equality wrapper constructor. - //! - //! @param __empty Empty sentinel value - //! @param __erased Erased sentinel value - //! @param __eq Equality binary callable - _CCCL_HOST_DEVICE_API constexpr __equal_wrapper(_Tp __empty, _Tp __erased, const _Equal& __eq) noexcept - : __empty_sentinel{__empty} - , __erased_sentinel{__erased} - , __equal{__eq} - {} - -#if _CCCL_CUDA_COMPILATION() - //! @brief Equality check with the given equality callable. - //! - //! @tparam _Lhs Left-hand side element type - //! @tparam _Rhs Right-hand side element type - //! - //! @param __lhs Left-hand side element to check equality - //! @param __rhs Right-hand side element to check equality - //! - //! @return `__equal` if `__lhs` and `__rhs` are equivalent, `__unequal` otherwise - template - [[nodiscard]] _CCCL_DEVICE_API constexpr __equal_result __equal_to(const _Lhs& __lhs, const _Rhs& __rhs) const noexcept - { - return __equal(__lhs, __rhs) ? __equal_result::__equal : __equal_result::__unequal; - } - - //! @brief Order-sensitive equality operator. - //! - //! @note This function always compares the right-hand side element against sentinel values first - //! then performs an equality check with the given `__equal` callable, i.e., `__equal(__lhs, __rhs)`. - //! @note Container (like set or map) slots MUST always be on the right-hand side. - //! - //! @tparam _IsInsert Flag indicating whether it's an insert equality check or not. Insert probing - //! stops when it's an empty or erased slot while query probing stops only when it's empty. - //! @tparam _Lhs Left-hand side element type - //! @tparam _Rhs Right-hand side element type - //! - //! @param __lhs Left-hand side element to check equality - //! @param __rhs Right-hand side element to check equality - //! - //! @return Three-way equality comparison result - template <__is_insert _IsInsert, class _Lhs, class _Rhs> - [[nodiscard]] _CCCL_DEVICE_API constexpr __equal_result operator()(const _Lhs& __lhs, const _Rhs& __rhs) const noexcept - { - if constexpr (_IsInsert == __is_insert::__yes) - { - if (detail::__bitwise_compare(__rhs, __empty_sentinel) || detail::__bitwise_compare(__rhs, __erased_sentinel)) - { - return __equal_result::__available; - } - else if constexpr (_AllowsDuplicates) - { - return __equal_result::__unequal; - } - else - { - return __equal_to(__lhs, __rhs); - } - } - else - { - return detail::__bitwise_compare(__rhs, __empty_sentinel) ? __equal_result::__empty : __equal_to(__lhs, __rhs); - } - } -#endif // _CCCL_CUDA_COMPILATION() -}; -} // namespace cuda::experimental::cuco::detail - -#include - -#endif // _CUDAX___CUCO_DETAIL_EQUAL_WRAPPER_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hash_functions/murmurhash3.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hash_functions/murmurhash3.cuh deleted file mode 100644 index 0ebaeeda..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hash_functions/murmurhash3.cuh +++ /dev/null @@ -1,848 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/* MurmurHash3_32 implementation from - * https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp - * ----------------------------------------------------------------------------- - * MurmurHash3 was written by Austin Appleby, and is placed in the public domain. The author - * hereby disclaims copyright to this source code. - * - * Note - The x86 and x64 versions do _not_ produce the same results, as the algorithms are - * optimized for their respective platforms. You can still compile and run any of them on any - * platform, but your performance with the non-native version will be less than optimal. - */ - -#ifndef _CUDAX___CUCO_DETAIL_HASH_FUNCTIONS_MURMURHASH3_CUH -#define _CUDAX___CUCO_DETAIL_HASH_FUNCTIONS_MURMURHASH3_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace cuda::experimental::cuco -{ -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint32_t -__fmix32(_Key __key, ::cuda::std::uint32_t __seed = 0) noexcept -{ - static_assert(sizeof(_Key) == 4, "Key type must be 4 bytes in size."); - - auto __h = ::cuda::std::bit_cast<::cuda::std::uint32_t>(__key) ^ __seed; - __h ^= __h >> 16; - __h *= 0x85ebca6b; - __h ^= __h >> 13; - __h *= 0xc2b2ae35; - __h ^= __h >> 16; - return __h; -} - -#if _CCCL_HAS_INT128() -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint64_t -__fmix64(_Key __key, ::cuda::std::uint64_t __seed = 0) noexcept -{ - static_assert(sizeof(_Key) == 8, "Key type must be 8 bytes in size."); - - auto __h = ::cuda::std::bit_cast<::cuda::std::uint64_t>(__key) ^ __seed; - __h ^= __h >> 33; - __h *= 0xff51afd7ed558ccdULL; - __h ^= __h >> 33; - __h *= 0xc4ceb9fe1a85ec53ULL; - __h ^= __h >> 33; - return __h; -} -#endif // _CCCL_HAS_INT128() - -//! @brief A `MurmurHash3_32` hash function to hash the given argument on host and device. -//! -//! @tparam _Key The type of the values to hash -template -struct _MurmurHash3_32 -{ - static constexpr ::cuda::std::uint32_t __c1 = 0xcc9e2d51; - static constexpr ::cuda::std::uint32_t __c2 = 0x1b873593; - - static constexpr ::cuda::std::uint32_t __block_size = 4; - static constexpr ::cuda::std::uint32_t __chunk_size = 4; - - _CCCL_HOST_DEVICE_API constexpr _MurmurHash3_32(::cuda::std::uint32_t __seed = 0) - : __seed_{__seed} - {} - - //! @brief Returns a hash value for its argument, as a value of type `::cuda::std::uint32_t`. - //! @param __key The input argument to hash - //! @return The resulting hash value - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint32_t operator()(const _Key& __key) const noexcept - { - using _Holder = _Byte_holder; - return __compute_hash(::cuda::std::bit_cast<_Holder>(__key)); - } - - //! @brief Returns a hash value for its argument, as a value of type `::cuda::std::uint32_t`. - //! @tparam _Extent The extent type - //! @param __keys span of keys to hash - //! @return The resulting hash value - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint64_t - operator()(::cuda::std::span<_Key, _Extent> __keys) const noexcept - { - return __compute_hash_span(__keys); - } - -private: - template - [[nodiscard]] _CCCL_HOST_DEVICE_API ::cuda::std::uint32_t __compute_hash(_Holder __holder) const noexcept - { - ::cuda::std::uint32_t __h1 = __seed_; - - //---------- - // body - if constexpr (_Holder::__num_blocks > 0) - { - ::cuda::static_for<_Holder::__num_blocks>([&](auto __i) { - ::cuda::std::uint32_t __k1 = __holder.__blocks[__i]; - __k1 *= __c1; - __k1 = ::cuda::std::rotl(__k1, 15); - __k1 *= __c2; - __h1 ^= __k1; - __h1 = ::cuda::std::rotl(__h1, 13); - __h1 = __h1 * 5 + 0xe6546b64; - }); - } - - //---------- - // tail - if constexpr (_Holder::__tail_size > 0) - { - ::cuda::std::uint32_t __k1 = 0; - switch (__holder.__tail_size) - { - case 3: - __k1 ^= ::cuda::std::to_integer<::cuda::std::uint32_t>(__holder.__bytes[2]) << 16; - [[fallthrough]]; - case 2: - __k1 ^= ::cuda::std::to_integer<::cuda::std::uint32_t>(__holder.__bytes[1]) << 8; - [[fallthrough]]; - case 1: - __k1 ^= ::cuda::std::to_integer<::cuda::std::uint32_t>(__holder.__bytes[0]); - __k1 *= __c1; - __k1 = ::cuda::std::rotl(__k1, 15); - __k1 *= __c2; - __h1 ^= __k1; - }; - } - - //---------- - // finalization - __h1 ^= ::cuda::std::uint32_t{sizeof(_Holder)}; - __h1 = ::cuda::experimental::cuco::__fmix32(__h1); - return __h1; - } - - [[nodiscard]] _CCCL_HOST_DEVICE_API ::cuda::std::uint32_t - __compute_hash_span(::cuda::std::span __keys) const noexcept - { - const auto __bytes = ::cuda::std::as_bytes(__keys).data(); - const auto __size = __keys.size_bytes(); - - const auto __nblocks = __size / __block_size; - - ::cuda::std::uint32_t __h1 = __seed_; - - //---------- - // body - for (::cuda::std::remove_const_t __i = 0; __i < __nblocks; __i++) - { - ::cuda::std::uint32_t __k1 = ::cuda::experimental::cuco::__load_chunk<::cuda::std::uint32_t>(__bytes, __i); - __k1 *= __c1; - __k1 = ::cuda::std::rotl(__k1, 15); - __k1 *= __c2; - __h1 ^= __k1; - __h1 = ::cuda::std::rotl(__h1, 13); - __h1 = __h1 * 5 + 0xe6546b64; - } - //---------- - // tail - ::cuda::std::uint32_t __k1 = 0; - switch (__size % 4) - { - case 3: - __k1 ^= ::cuda::std::to_integer<::cuda::std::uint32_t>(__bytes[__nblocks * __block_size + 2]) << 16; - [[fallthrough]]; - case 2: - __k1 ^= ::cuda::std::to_integer<::cuda::std::uint32_t>(__bytes[__nblocks * __block_size + 1]) << 8; - [[fallthrough]]; - case 1: - __k1 ^= ::cuda::std::to_integer<::cuda::std::uint32_t>(__bytes[__nblocks * __block_size + 0]); - __k1 *= __c1; - __k1 = ::cuda::std::rotl(__k1, 15); - __k1 *= __c2; - __h1 ^= __k1; - }; - //---------- - // finalization - __h1 ^= __size; - __h1 = ::cuda::experimental::cuco::__fmix32(__h1); - return __h1; - } - - ::cuda::std::uint32_t __seed_; -}; - -#if _CCCL_HAS_INT128() - -template -struct _MurmurHash3_x86_128 -{ -private: - static constexpr ::cuda::std::uint32_t __c1 = 0x239b961b; - static constexpr ::cuda::std::uint32_t __c2 = 0xab0e9789; - static constexpr ::cuda::std::uint32_t __c3 = 0x38b34ae5; - static constexpr ::cuda::std::uint32_t __c4 = 0xa1e38b93; - - static constexpr ::cuda::std::uint32_t __block_size = 4; - static constexpr ::cuda::std::uint32_t __chunk_size = 16; - -public: - _CCCL_HOST_DEVICE_API constexpr _MurmurHash3_x86_128(::cuda::std::uint32_t __seed = 0) - : __seed_{__seed} - {} - - //! @brief Returns a hash value for its argument, as a value of type `__uint128_t`. - //! @param __key The input argument to hash - //! @return The resulting hash value - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __uint128_t operator()(const _Key& __key) const noexcept - { - using _Holder = _Byte_holder; - return __compute_hash(::cuda::std::bit_cast<_Holder>(__key)); - } - - //! @brief Returns a hash value for its argument, as a value of type `__uint128_t`. - //! @tparam _Extent The extent type - //! @param __keys span of keys to hash - //! @return The resulting hash value - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __uint128_t - operator()(::cuda::std::span<_Key, _Extent> __keys) const noexcept - { - return __compute_hash_span(__keys); - } - -private: - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __uint128_t __compute_hash(_Holder __holder) const noexcept - { - ::cuda::std::array<::cuda::std::uint32_t, 4> __h{__seed_, __seed_, __seed_, __seed_}; - const auto __size = ::cuda::std::uint32_t{sizeof(_Holder)}; - - if constexpr (_Holder::__num_chunks > 0) - { - ::cuda::static_for<_Holder::__num_chunks>([&](auto __i) { - ::cuda::std::uint32_t __k1 = __holder.__blocks[4 * __i]; - ::cuda::std::uint32_t __k2 = __holder.__blocks[4 * __i + 1]; - ::cuda::std::uint32_t __k3 = __holder.__blocks[4 * __i + 2]; - ::cuda::std::uint32_t __k4 = __holder.__blocks[4 * __i + 3]; - - __k1 *= __c1; - __k1 = ::cuda::std::rotl(__k1, 15); - __k1 *= __c2; - __h[0] ^= __k1; - - __h[0] = ::cuda::std::rotl(__h[0], 19); - __h[0] += __h[1]; - __h[0] = __h[0] * 5 + 0x561ccd1b; - - __k2 *= __c2; - __k2 = ::cuda::std::rotl(__k2, 16); - __k2 *= __c3; - __h[1] ^= __k2; - - __h[1] = ::cuda::std::rotl(__h[1], 17); - __h[1] += __h[2]; - __h[1] = __h[1] * 5 + 0x0bcaa747; - - __k3 *= __c3; - __k3 = ::cuda::std::rotl(__k3, 17); - __k3 *= __c4; - __h[2] ^= __k3; - - __h[2] = ::cuda::std::rotl(__h[2], 15); - __h[2] += __h[3]; - __h[2] = __h[2] * 5 + 0x96cd1c35; - - __k4 *= __c4; - __k4 = ::cuda::std::rotl(__k4, 18); - __k4 *= __c1; - __h[3] ^= __k4; - - __h[3] = ::cuda::std::rotl(__h[3], 13); - __h[3] += __h[0]; - __h[3] = __h[3] * 5 + 0x32ac3b17; - }); - } - // tail - if constexpr (_Holder::__tail_size > 0) - { - ::cuda::std::uint32_t __k1 = 0; - ::cuda::std::uint32_t __k2 = 0; - ::cuda::std::uint32_t __k3 = 0; - ::cuda::std::uint32_t __k4 = 0; - - const auto __tail = __holder.__bytes; - switch (__size % __chunk_size) - { - case 15: - __k4 ^= static_cast<::cuda::std::uint32_t>(__tail[14]) << 16; - [[fallthrough]]; - case 14: - __k4 ^= static_cast<::cuda::std::uint32_t>(__tail[13]) << 8; - [[fallthrough]]; - case 13: - __k4 ^= static_cast<::cuda::std::uint32_t>(__tail[12]) << 0; - __k4 *= __c4; - __k4 = ::cuda::std::rotl(__k4, 18); - __k4 *= __c1; - __h[3] ^= __k4; - [[fallthrough]]; - - case 12: - __k3 ^= static_cast<::cuda::std::uint32_t>(__tail[11]) << 24; - [[fallthrough]]; - case 11: - __k3 ^= static_cast<::cuda::std::uint32_t>(__tail[10]) << 16; - [[fallthrough]]; - case 10: - __k3 ^= static_cast<::cuda::std::uint32_t>(__tail[9]) << 8; - [[fallthrough]]; - case 9: - __k3 ^= static_cast<::cuda::std::uint32_t>(__tail[8]) << 0; - __k3 *= __c3; - __k3 = ::cuda::std::rotl(__k3, 17); - __k3 *= __c4; - __h[2] ^= __k3; - [[fallthrough]]; - - case 8: - __k2 ^= static_cast<::cuda::std::uint32_t>(__tail[7]) << 24; - [[fallthrough]]; - case 7: - __k2 ^= static_cast<::cuda::std::uint32_t>(__tail[6]) << 16; - [[fallthrough]]; - case 6: - __k2 ^= static_cast<::cuda::std::uint32_t>(__tail[5]) << 8; - [[fallthrough]]; - case 5: - __k2 ^= static_cast<::cuda::std::uint32_t>(__tail[4]) << 0; - __k2 *= __c2; - __k2 = ::cuda::std::rotl(__k2, 16); - __k2 *= __c3; - __h[1] ^= __k2; - [[fallthrough]]; - - case 4: - __k1 ^= static_cast<::cuda::std::uint32_t>(__tail[3]) << 24; - [[fallthrough]]; - case 3: - __k1 ^= static_cast<::cuda::std::uint32_t>(__tail[2]) << 16; - [[fallthrough]]; - case 2: - __k1 ^= static_cast<::cuda::std::uint32_t>(__tail[1]) << 8; - [[fallthrough]]; - case 1: - __k1 ^= static_cast<::cuda::std::uint32_t>(__tail[0]) << 0; - __k1 *= __c1; - __k1 = ::cuda::std::rotl(__k1, 15); - __k1 *= __c2; - __h[0] ^= __k1; - }; - } - - // finalization - __h[0] ^= __size; - __h[1] ^= __size; - __h[2] ^= __size; - __h[3] ^= __size; - - __h[0] += __h[1]; - __h[0] += __h[2]; - __h[0] += __h[3]; - __h[1] += __h[0]; - __h[2] += __h[0]; - __h[3] += __h[0]; - - __h[0] = ::cuda::experimental::cuco::__fmix32(__h[0]); - __h[1] = ::cuda::experimental::cuco::__fmix32(__h[1]); - __h[2] = ::cuda::experimental::cuco::__fmix32(__h[2]); - __h[3] = ::cuda::experimental::cuco::__fmix32(__h[3]); - - __h[0] += __h[1]; - __h[0] += __h[2]; - __h[0] += __h[3]; - __h[1] += __h[0]; - __h[2] += __h[0]; - __h[3] += __h[0]; - - return ::cuda::std::bit_cast<__uint128_t>(__h); - } - - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __uint128_t - __compute_hash_span(::cuda::std::span __keys) const noexcept - { - const auto __bytes = ::cuda::std::as_bytes(__keys).data(); - const auto __size = __keys.size_bytes(); - - const auto __nchunks = __size / __chunk_size; - - ::cuda::std::array<::cuda::std::uint32_t, 4> __h{__seed_, __seed_, __seed_, __seed_}; - - // body - for (::cuda::std::remove_const_t __i = 0; __size >= __chunk_size && __i < __nchunks; ++__i) - { - ::cuda::std::uint32_t __k1 = ::cuda::experimental::cuco::__load_chunk<::cuda::std::uint32_t>(__bytes, 4 * __i); - ::cuda::std::uint32_t __k2 = - ::cuda::experimental::cuco::__load_chunk<::cuda::std::uint32_t>(__bytes, 4 * __i + 1); - ::cuda::std::uint32_t __k3 = - ::cuda::experimental::cuco::__load_chunk<::cuda::std::uint32_t>(__bytes, 4 * __i + 2); - ::cuda::std::uint32_t __k4 = - ::cuda::experimental::cuco::__load_chunk<::cuda::std::uint32_t>(__bytes, 4 * __i + 3); - - __k1 *= __c1; - __k1 = ::cuda::std::rotl(__k1, 15); - __k1 *= __c2; - __h[0] ^= __k1; - - __h[0] = ::cuda::std::rotl(__h[0], 19); - __h[0] += __h[1]; - __h[0] = __h[0] * 5 + 0x561ccd1b; - - __k2 *= __c2; - __k2 = ::cuda::std::rotl(__k2, 16); - __k2 *= __c3; - __h[1] ^= __k2; - - __h[1] = ::cuda::std::rotl(__h[1], 17); - __h[1] += __h[2]; - __h[1] = __h[1] * 5 + 0x0bcaa747; - - __k3 *= __c3; - __k3 = ::cuda::std::rotl(__k3, 17); - __k3 *= __c4; - __h[2] ^= __k3; - - __h[2] = ::cuda::std::rotl(__h[2], 15); - __h[2] += __h[3]; - __h[2] = __h[2] * 5 + 0x96cd1c35; - - __k4 *= __c4; - __k4 = ::cuda::std::rotl(__k4, 18); - __k4 *= __c1; - __h[3] ^= __k4; - - __h[3] = ::cuda::std::rotl(__h[3], 13); - __h[3] += __h[0]; - __h[3] = __h[3] * 5 + 0x32ac3b17; - } - - // tail - ::cuda::std::uint32_t __k1 = 0; - ::cuda::std::uint32_t __k2 = 0; - ::cuda::std::uint32_t __k3 = 0; - ::cuda::std::uint32_t __k4 = 0; - - const auto __tail = __bytes + __nchunks * __chunk_size; - - switch (__size % __chunk_size) - { - case 15: - __k4 ^= static_cast<::cuda::std::uint32_t>(__tail[14]) << 16; - [[fallthrough]]; - case 14: - __k4 ^= static_cast<::cuda::std::uint32_t>(__tail[13]) << 8; - [[fallthrough]]; - case 13: - __k4 ^= static_cast<::cuda::std::uint32_t>(__tail[12]) << 0; - __k4 *= __c4; - __k4 = ::cuda::std::rotl(__k4, 18); - __k4 *= __c1; - __h[3] ^= __k4; - [[fallthrough]]; - - case 12: - __k3 ^= static_cast<::cuda::std::uint32_t>(__tail[11]) << 24; - [[fallthrough]]; - case 11: - __k3 ^= static_cast<::cuda::std::uint32_t>(__tail[10]) << 16; - [[fallthrough]]; - case 10: - __k3 ^= static_cast<::cuda::std::uint32_t>(__tail[9]) << 8; - [[fallthrough]]; - case 9: - __k3 ^= static_cast<::cuda::std::uint32_t>(__tail[8]) << 0; - __k3 *= __c3; - __k3 = ::cuda::std::rotl(__k3, 17); - __k3 *= __c4; - __h[2] ^= __k3; - [[fallthrough]]; - - case 8: - __k2 ^= static_cast<::cuda::std::uint32_t>(__tail[7]) << 24; - [[fallthrough]]; - case 7: - __k2 ^= static_cast<::cuda::std::uint32_t>(__tail[6]) << 16; - [[fallthrough]]; - case 6: - __k2 ^= static_cast<::cuda::std::uint32_t>(__tail[5]) << 8; - [[fallthrough]]; - case 5: - __k2 ^= static_cast<::cuda::std::uint32_t>(__tail[4]) << 0; - __k2 *= __c2; - __k2 = ::cuda::std::rotl(__k2, 16); - __k2 *= __c3; - __h[1] ^= __k2; - [[fallthrough]]; - - case 4: - __k1 ^= static_cast<::cuda::std::uint32_t>(__tail[3]) << 24; - [[fallthrough]]; - case 3: - __k1 ^= static_cast<::cuda::std::uint32_t>(__tail[2]) << 16; - [[fallthrough]]; - case 2: - __k1 ^= static_cast<::cuda::std::uint32_t>(__tail[1]) << 8; - [[fallthrough]]; - case 1: - __k1 ^= static_cast<::cuda::std::uint32_t>(__tail[0]) << 0; - __k1 *= __c1; - __k1 = ::cuda::std::rotl(__k1, 15); - __k1 *= __c2; - __h[0] ^= __k1; - }; - - // finalization - __h[0] ^= __size; - __h[1] ^= __size; - __h[2] ^= __size; - __h[3] ^= __size; - - __h[0] += __h[1]; - __h[0] += __h[2]; - __h[0] += __h[3]; - __h[1] += __h[0]; - __h[2] += __h[0]; - __h[3] += __h[0]; - - __h[0] = ::cuda::experimental::cuco::__fmix32(__h[0]); - __h[1] = ::cuda::experimental::cuco::__fmix32(__h[1]); - __h[2] = ::cuda::experimental::cuco::__fmix32(__h[2]); - __h[3] = ::cuda::experimental::cuco::__fmix32(__h[3]); - - __h[0] += __h[1]; - __h[0] += __h[2]; - __h[0] += __h[3]; - __h[1] += __h[0]; - __h[2] += __h[0]; - __h[3] += __h[0]; - - return ::cuda::std::bit_cast<__uint128_t>(__h); - } - -private: - ::cuda::std::uint32_t __seed_; -}; - -template -struct _MurmurHash3_x64_128 -{ -private: - static constexpr ::cuda::std::uint64_t __c1 = 0x87c37b91114253d5ull; - static constexpr ::cuda::std::uint64_t __c2 = 0x4cf5ad432745937full; - - static constexpr ::cuda::std::uint32_t __block_size = 8; - static constexpr ::cuda::std::uint32_t __chunk_size = 16; - -public: - _CCCL_HOST_DEVICE_API constexpr _MurmurHash3_x64_128(::cuda::std::uint64_t __seed = 0) - : __seed_{__seed} - {} - - //! @brief Returns a hash value for its argument, as a value of type `__uint128_t`. - //! @param __key The input argument to hash - //! @return The resulting hash value - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __uint128_t operator()(const _Key& __key) const noexcept - { - using _Holder = _Byte_holder; - return __compute_hash(::cuda::std::bit_cast<_Holder>(__key)); - } - - //! @brief Returns a hash value for its argument, as a value of type `__uint128_t`. - //! @tparam _Extent The extent type - //! @param __keys span of keys to hash - //! @return The resulting hash value - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __uint128_t - operator()(::cuda::std::span<_Key, _Extent> __keys) const noexcept - { - return __compute_hash_span(__keys); - } - -private: - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __uint128_t __compute_hash(_Holder __holder) const noexcept - { - ::cuda::std::array<::cuda::std::uint64_t, 2> __h{__seed_, __seed_}; - const auto __size = ::cuda::std::uint64_t{sizeof(_Holder)}; - - if constexpr (_Holder::__num_chunks > 0) - { - ::cuda::static_for<_Holder::__num_chunks>([&](auto __i) { - ::cuda::std::uint64_t __k1 = __holder.__blocks[2 * __i]; - ::cuda::std::uint64_t __k2 = __holder.__blocks[2 * __i + 1]; - - __k1 *= __c1; - __k1 = ::cuda::std::rotl(__k1, 31); - __k1 *= __c2; - __h[0] ^= __k1; - - __h[0] = ::cuda::std::rotl(__h[0], 27); - __h[0] += __h[1]; - __h[0] = __h[0] * 5 + 0x52dce729; - - __k2 *= __c2; - __k2 = ::cuda::std::rotl(__k2, 33); - __k2 *= __c1; - __h[1] ^= __k2; - - __h[1] = ::cuda::std::rotl(__h[1], 31); - __h[1] += __h[0]; - __h[1] = __h[1] * 5 + 0x38495ab5; - }); - } - // tail - if constexpr (_Holder::__tail_size > 0) - { - ::cuda::std::uint64_t __k1 = 0; - ::cuda::std::uint64_t __k2 = 0; - - const auto __tail = __holder.__bytes; - switch (__size % __chunk_size) - { - case 15: - __k2 ^= static_cast<::cuda::std::uint64_t>(__tail[14]) << 48; - [[fallthrough]]; - case 14: - __k2 ^= static_cast<::cuda::std::uint64_t>(__tail[13]) << 40; - [[fallthrough]]; - case 13: - __k2 ^= static_cast<::cuda::std::uint64_t>(__tail[12]) << 32; - [[fallthrough]]; - case 12: - __k2 ^= static_cast<::cuda::std::uint64_t>(__tail[11]) << 24; - [[fallthrough]]; - case 11: - __k2 ^= static_cast<::cuda::std::uint64_t>(__tail[10]) << 16; - [[fallthrough]]; - case 10: - __k2 ^= static_cast<::cuda::std::uint64_t>(__tail[9]) << 8; - [[fallthrough]]; - case 9: - __k2 ^= static_cast<::cuda::std::uint64_t>(__tail[8]) << 0; - __k2 *= __c2; - __k2 = ::cuda::std::rotl(__k2, 33); - __k2 *= __c1; - __h[1] ^= __k2; - [[fallthrough]]; - case 8: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[7]) << 56; - [[fallthrough]]; - case 7: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[6]) << 48; - [[fallthrough]]; - case 6: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[5]) << 40; - [[fallthrough]]; - case 5: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[4]) << 32; - [[fallthrough]]; - case 4: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[3]) << 24; - [[fallthrough]]; - case 3: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[2]) << 16; - [[fallthrough]]; - case 2: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[1]) << 8; - [[fallthrough]]; - case 1: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[0]) << 0; - __k1 *= __c1; - __k1 = ::cuda::std::rotl(__k1, 31); - __k1 *= __c2; - __h[0] ^= __k1; - } - } - - // finalization - __h[0] ^= __size; - __h[1] ^= __size; - - __h[0] += __h[1]; - __h[1] += __h[0]; - - __h[0] = ::cuda::experimental::cuco::__fmix64(__h[0]); - __h[1] = ::cuda::experimental::cuco::__fmix64(__h[1]); - - __h[0] += __h[1]; - __h[1] += __h[0]; - - return ::cuda::std::bit_cast<__uint128_t>(__h); - } - - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __uint128_t - __compute_hash_span(::cuda::std::span __keys) const noexcept - { - const auto __bytes = ::cuda::std::as_bytes(__keys).data(); - const auto __size = __keys.size_bytes(); - - const auto __nchunks = __size / __chunk_size; - - ::cuda::std::array<::cuda::std::uint64_t, 2> __h{__seed_, __seed_}; - - // body - for (::cuda::std::remove_const_t __i = 0; __size >= __chunk_size && __i < __nchunks; ++__i) - { - ::cuda::std::uint64_t __k1 = ::cuda::experimental::cuco::__load_chunk<::cuda::std::uint64_t>(__bytes, 2 * __i); - ::cuda::std::uint64_t __k2 = - ::cuda::experimental::cuco::__load_chunk<::cuda::std::uint64_t>(__bytes, 2 * __i + 1); - - __k1 *= __c1; - __k1 = ::cuda::std::rotl(__k1, 31); - __k1 *= __c2; - - __h[0] ^= __k1; - __h[0] = ::cuda::std::rotl(__h[0], 27); - __h[0] += __h[1]; - __h[0] = __h[0] * 5 + 0x52dce729; - - __k2 *= __c2; - __k2 = ::cuda::std::rotl(__k2, 33); - __k2 *= __c1; - - __h[1] ^= __k2; - __h[1] = ::cuda::std::rotl(__h[1], 31); - __h[1] += __h[0]; - __h[1] = __h[1] * 5 + 0x38495ab5; - } - - // tail - ::cuda::std::uint64_t __k1 = 0; - ::cuda::std::uint64_t __k2 = 0; - const auto __tail = __bytes + __nchunks * __chunk_size; - - switch (__size % __chunk_size) - { - case 15: - __k2 ^= static_cast<::cuda::std::uint64_t>(__tail[14]) << 48; - [[fallthrough]]; - case 14: - __k2 ^= static_cast<::cuda::std::uint64_t>(__tail[13]) << 40; - [[fallthrough]]; - case 13: - __k2 ^= static_cast<::cuda::std::uint64_t>(__tail[12]) << 32; - [[fallthrough]]; - case 12: - __k2 ^= static_cast<::cuda::std::uint64_t>(__tail[11]) << 24; - [[fallthrough]]; - case 11: - __k2 ^= static_cast<::cuda::std::uint64_t>(__tail[10]) << 16; - [[fallthrough]]; - case 10: - __k2 ^= static_cast<::cuda::std::uint64_t>(__tail[9]) << 8; - [[fallthrough]]; - case 9: - __k2 ^= static_cast<::cuda::std::uint64_t>(__tail[8]) << 0; - __k2 *= __c2; - __k2 = ::cuda::std::rotl(__k2, 33); - __k2 *= __c1; - __h[1] ^= __k2; - [[fallthrough]]; - - case 8: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[7]) << 56; - [[fallthrough]]; - case 7: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[6]) << 48; - [[fallthrough]]; - case 6: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[5]) << 40; - [[fallthrough]]; - case 5: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[4]) << 32; - [[fallthrough]]; - case 4: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[3]) << 24; - [[fallthrough]]; - case 3: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[2]) << 16; - [[fallthrough]]; - case 2: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[1]) << 8; - [[fallthrough]]; - case 1: - __k1 ^= static_cast<::cuda::std::uint64_t>(__tail[0]) << 0; - __k1 *= __c1; - __k1 = ::cuda::std::rotl(__k1, 31); - __k1 *= __c2; - __h[0] ^= __k1; - }; - - // finalization - __h[0] ^= __size; - __h[1] ^= __size; - - __h[0] += __h[1]; - __h[1] += __h[0]; - - __h[0] = ::cuda::experimental::cuco::__fmix64(__h[0]); - __h[1] = ::cuda::experimental::cuco::__fmix64(__h[1]); - - __h[0] += __h[1]; - __h[1] += __h[0]; - - return ::cuda::std::bit_cast<__uint128_t>(__h); - } - -private: - ::cuda::std::uint64_t __seed_; -}; - -#endif // _CCCL_HAS_INT128() -} // namespace cuda::experimental::cuco - -#include - -#endif // _CUDAX___CUCO_DETAIL_HASH_FUNCTIONS_MURMURHASH3_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hash_functions/utils.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hash_functions/utils.cuh deleted file mode 100644 index b0052fb8..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hash_functions/utils.cuh +++ /dev/null @@ -1,150 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_HASH_FUNCTIONS_UTILS_CUH -#define _CUDAX___CUCO_DETAIL_HASH_FUNCTIONS_UTILS_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include - -#include - -namespace cuda::experimental::cuco -{ -//! @brief Loads a chunk of type _Tp from a byte pointer at a given index, handling alignment -//! -//! @tparam _Tp The type of the chunk to load (must be 4 or 8 bytes) -//! @tparam _Extent The index type -//! @param __bytes Pointer to the byte array -//! @param __index The index of the chunk to load -//! @return The loaded chunk of type _Tp -template -[[nodiscard]] _CCCL_HOST_DEVICE_API _Tp __load_chunk(::cuda::std::byte const* const __bytes, _Extent __index) noexcept -{ - static_assert(sizeof(_Tp) == 4 || sizeof(_Tp) == 8, "__load_chunk must be used with types of size 4 or 8 bytes"); - - const auto __ptr = __bytes + __index * sizeof(_Tp); - - _Tp __chunk; - if constexpr (alignof(_Tp) == 8) - { - if (::cuda::is_aligned(__ptr, 8)) - { - ::cuda::std::memcpy(&__chunk, ::cuda::std::assume_aligned<8>(__ptr), sizeof(_Tp)); - return __chunk; - } - } - - if (::cuda::is_aligned(__ptr, 4)) - { - ::cuda::std::memcpy(&__chunk, ::cuda::std::assume_aligned<4>(__ptr), sizeof(_Tp)); - } - else if (::cuda::is_aligned(__ptr, 2)) - { - ::cuda::std::memcpy(&__chunk, ::cuda::std::assume_aligned<2>(__ptr), sizeof(_Tp)); - } - else - { - ::cuda::std::memcpy(&__chunk, __ptr, sizeof(_Tp)); - } - return __chunk; -} - -//! @brief Type erased holder of all the bytes -//! -//! @tparam _KeySize The size of the key in bytes -//! @tparam _ChunkSize The size of a chunk in bytes -//! @tparam _BlockSize The size of a block in bytes (same as sizeof(_BlockT)) -//! @tparam _UseTailBlock Whether to use a tail block for the last bytes -//! @tparam _BlockT The type of the block -//! @tparam _HasBlocksOrChunks Whether the key size is larger than the chunk size or block size -//! @tparam _HasTail Whether the key size is larger than the block size -//! -//! @note _UseTailBlock is true for xxhash and false for murmurhash, as xxhash consider's tail as blocks for the last -//! bytes, where as murmurhash considers the tail as a bytes -template = _BlockSize) : (_KeySize >= _ChunkSize), - bool _HasTail = _UseTailBlock ? ((_KeySize % _BlockSize) != 0) : ((_KeySize % _ChunkSize) != 0)> -struct _Byte_holder -{ - //! The number of trailing bytes that do not fit into a _BlockT - static constexpr size_t __tail_size = _UseTailBlock ? _KeySize % _BlockSize : _KeySize % _ChunkSize; - - //! The number of `_ChunkSize` chunks - static constexpr size_t __num_chunks = _KeySize / _ChunkSize; - - //! The number of `_BlockSize` blocks in a `_ChunkSize` chunk - static constexpr size_t __blocks_per_chunk = _ChunkSize / _BlockSize; - - //! The number of `_BlockSize` blocks - static constexpr size_t __num_blocks = _UseTailBlock ? _KeySize / _BlockSize : __num_chunks * __blocks_per_chunk; - - _BlockT __blocks[__num_blocks]; - ::cuda::std::byte __bytes[__tail_size]; -}; - -//! @brief Type erased holder of small types < _BlockSize -template -struct _Byte_holder<_KeySize, _ChunkSize, _BlockSize, _UseTailBlock, _BlockT, false, true> -{ - //! The number of trailing bytes that do not fit into a _BlockT - static constexpr size_t __tail_size = _UseTailBlock ? _KeySize % _BlockSize : _KeySize % _ChunkSize; - - //! The number of `_ChunkSize` chunks - static constexpr size_t __num_chunks = _KeySize / _ChunkSize; - - //! The number of `_BlockSize` blocks in a `_ChunkSize` chunk - static constexpr size_t __blocks_per_chunk = _ChunkSize / _BlockSize; - - //! The number of `_BlockSize` blocks in a `_ChunkSize` chunk - static constexpr size_t __num_blocks = _UseTailBlock ? _KeySize / _BlockSize : __num_chunks * __blocks_per_chunk; - - ::cuda::std::byte __bytes[__tail_size]; -}; - -//! @brief Type erased holder of types without trailing bytes -template -struct _Byte_holder<_KeySize, _ChunkSize, _BlockSize, _UseTailBlock, _BlockT, true, false> -{ - //! The number of trailing bytes that do not fit into a _BlockT - static constexpr size_t __tail_size = _UseTailBlock ? _KeySize % _BlockSize : _KeySize % _ChunkSize; - - //! The number of `_ChunkSize` chunks - static constexpr size_t __num_chunks = _KeySize / _ChunkSize; - - //! The number of `_BlockSize` blocks in a `_ChunkSize` chunk - static constexpr size_t __blocks_per_chunk = _ChunkSize / _BlockSize; - - //! The number of `_BlockSize` blocks - static constexpr size_t __num_blocks = _UseTailBlock ? _KeySize / _BlockSize : __num_chunks * __blocks_per_chunk; - - _BlockT __blocks[__num_blocks]; -}; -} // namespace cuda::experimental::cuco - -#include - -#endif // _CUDAX___CUCO_DETAIL_HASH_FUNCTIONS_UTILS_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hash_functions/xxhash.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hash_functions/xxhash.cuh deleted file mode 100644 index 19353ba9..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hash_functions/xxhash.cuh +++ /dev/null @@ -1,426 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -/* - * `_XXHash_32` and `_XXHash_64` implementation from - * https://github.com/Cyan4973/xxHash - * ----------------------------------------------------------------------------- - * xxHash - Extremely Fast Hash algorithm - * Header File - * Copyright (C) 2012-2021 Yann Collet - * - * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _CUDAX___CUCO_DETAIL_HASH_FUNCTIONS_XXHASH_CUH -#define _CUDAX___CUCO_DETAIL_HASH_FUNCTIONS_XXHASH_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace cuda::experimental::cuco -{ -//! @brief A `_XXHash_32` hash function to hash the given argument on host and device. -//! -//! @tparam Key The type of the values to hash -template -struct _XXHash_32 -{ -private: - static constexpr ::cuda::std::uint32_t __prime1 = 0x9e3779b1u; - static constexpr ::cuda::std::uint32_t __prime2 = 0x85ebca77u; - static constexpr ::cuda::std::uint32_t __prime3 = 0xc2b2ae3du; - static constexpr ::cuda::std::uint32_t __prime4 = 0x27d4eb2fu; - static constexpr ::cuda::std::uint32_t __prime5 = 0x165667b1u; - - static constexpr ::cuda::std::uint32_t __block_size = 4; - static constexpr ::cuda::std::uint32_t __chunk_size = 16; - -public: - //! @brief Constructs a XXH32 hash function with the given `seed`. - //! @param seed A custom number to randomize the resulting hash value - _CCCL_HOST_DEVICE_API constexpr _XXHash_32(::cuda::std::uint32_t __seed = 0) - : __seed_{__seed} - {} - - //! @brief Returns a hash value for its argument, as a value of type `::cuda::std::uint32_t`. - //! @param __key The input argument to hash - //! @return The resulting hash value for `__key` - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint32_t operator()(const _Key& __key) const noexcept - { - using _Holder = _Byte_holder; - // explicit copy to avoid emitting a bunch of LDG.8 instructions - const _Key __copy{__key}; - return __compute_hash(::cuda::std::bit_cast<_Holder>(__copy)); - } - - //! @brief Returns a hash value for its argument, as a value of type `::cuda::std::uint32_t`. - //! @tparam _Extent The extent type - //! @param __keys span of keys to hash - //! @return The resulting hash value - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint32_t - operator()(::cuda::std::span<_Key, _Extent> __keys) const noexcept - { - return __compute_hash_span(__keys); - } - -private: - //! @brief Returns a hash value for its argument, as a value of type `::cuda::std::uint32_t`. - //! - //! @tparam _Extent The extent type - //! @param __holder The input argument to hash in form of a byte holder - //! @return The resulting hash value - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint32_t __compute_hash(_Holder __holder) const noexcept - { - ::cuda::std::uint32_t __offset = 0; - ::cuda::std::uint32_t __h32 = {}; - - // process data in 16-byte chunks - if constexpr (_Holder::__num_chunks > 0) - { - ::cuda::std::array<::cuda::std::uint32_t, 4> __v; - __v[0] = __seed_ + __prime1 + __prime2; - __v[1] = __seed_ + __prime2; - __v[2] = __seed_; - __v[3] = __seed_ - __prime1; - - for (::cuda::std::uint32_t __i = 0; __i < _Holder::__num_chunks; ++__i) - { - ::cuda::static_for<4>([&](auto i) { - __v[i] += __holder.__blocks[__offset++] * __prime2; - __v[i] = ::cuda::std::rotl(__v[i], 13); - __v[i] *= __prime1; - }); - } - __h32 = ::cuda::std::rotl(__v[0], 1) + ::cuda::std::rotl(__v[1], 7) + ::cuda::std::rotl(__v[2], 12) - + ::cuda::std::rotl(__v[3], 18); - } - else - { - __h32 = __seed_ + __prime5; - } - - __h32 += ::cuda::std::uint32_t{sizeof(_Holder)}; - - // remaining data can be processed in 4-byte chunks - if constexpr (_Holder::__num_blocks % __chunk_size > 0) - { - for (; __offset < _Holder::__num_blocks; ++__offset) - { - __h32 += __holder.__blocks[__offset] * __prime3; - __h32 = ::cuda::std::rotl(__h32, 17) * __prime4; - } - } - - // the following loop is only needed if the size of the key is not a multiple of the block size - if constexpr (_Holder::__tail_size > 0) - { - for (::cuda::std::uint32_t __i = 0; __i < _Holder::__tail_size; ++__i) - { - __h32 += (static_cast<::cuda::std::uint32_t>(__holder.__bytes[__i])) * __prime5; - __h32 = ::cuda::std::rotl(__h32, 11) * __prime1; - } - } - - return __finalize(__h32); - } - - //! @brief Returns a hash value for its argument, as a value of type `::cuda::std::uint32_t`. - //! - //! @tparam _Extent The extent type - //! @param __holder The input argument to hash in form of a span - //! @return The resulting hash value - [[nodiscard]] _CCCL_HOST_DEVICE_API ::cuda::std::uint32_t - __compute_hash_span(::cuda::std::span<_Key> __keys) const noexcept - { - auto __bytes = ::cuda::std::as_bytes(__keys).data(); - const auto __size = __keys.size_bytes(); - - ::cuda::std::uint32_t __offset = 0; - ::cuda::std::uint32_t __h32 = {}; - - // data can be processed in 16-byte chunks - if (__size >= 16) - { - const auto __limit = __size - 16; - ::cuda::std::array<::cuda::std::uint32_t, 4> __v; - - __v[0] = __seed_ + __prime1 + __prime2; - __v[1] = __seed_ + __prime2; - __v[2] = __seed_; - __v[3] = __seed_ - __prime1; - - for (; __offset <= __limit; __offset += 16) - { - // pipeline 4*4byte computations - const auto __pipeline_offset = __offset / 4; - ::cuda::static_for<4>([&](auto i) { - __v[i] += ::cuda::experimental::cuco::__load_chunk<::cuda::std::uint32_t>(__bytes, __pipeline_offset + i) - * __prime2; - __v[i] = ::cuda::std::rotl(__v[i], 13); - __v[i] *= __prime1; - }); - } - - __h32 = ::cuda::std::rotl(__v[0], 1) + ::cuda::std::rotl(__v[1], 7) + ::cuda::std::rotl(__v[2], 12) - + ::cuda::std::rotl(__v[3], 18); - } - else - { - __h32 = __seed_ + __prime5; - } - - __h32 += __size; - - // remaining data can be processed in 4-byte chunks - if ((__size % 16) >= 4) - { - _CCCL_PRAGMA_UNROLL(4) - for (; __offset <= __size - 4; __offset += 4) - { - __h32 += ::cuda::experimental::cuco::__load_chunk<::cuda::std::uint32_t>(__bytes, __offset / 4) * __prime3; - __h32 = ::cuda::std::rotl(__h32, 17) * __prime4; - } - } - - // the following loop is only needed if the size of the key is not a multiple of the block size - if (__size % 4) - { - while (__offset < __size) - { - __h32 += (::cuda::std::to_integer<::cuda::std::uint32_t>(__bytes[__offset]) & 255) * __prime5; - __h32 = ::cuda::std::rotl(__h32, 11) * __prime1; - ++__offset; - } - } - - return __finalize(__h32); - } - - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint32_t - __finalize(::cuda::std::uint32_t __h) const noexcept - { - __h ^= __h >> 15; - __h *= __prime2; - __h ^= __h >> 13; - __h *= __prime3; - __h ^= __h >> 16; - return __h; - } - - ::cuda::std::uint32_t __seed_; -}; - -//! @brief A `XXHash_64` hash function to hash the given argument on host and device. -//! -//! @tparam _Key The type of the values to hash -template -struct _XXHash_64 -{ -private: - static constexpr ::cuda::std::uint64_t __prime1 = 11400714785074694791ull; - static constexpr ::cuda::std::uint64_t __prime2 = 14029467366897019727ull; - static constexpr ::cuda::std::uint64_t __prime3 = 1609587929392839161ull; - static constexpr ::cuda::std::uint64_t __prime4 = 9650029242287828579ull; - static constexpr ::cuda::std::uint64_t __prime5 = 2870177450012600261ull; - -public: - //! @brief Constructs a XXH64 hash function with the given `seed`. - //! - //! @param seed A custom number to randomize the resulting hash value - _CCCL_HOST_DEVICE_API constexpr _XXHash_64(::cuda::std::uint64_t __seed = 0) - : __seed_{__seed} - {} - - //! @brief Returns a hash value for its argument, as a value of type `result_type`. - //! - //! @param _Key The input argument to hash - //! @return The resulting hash value for `key` - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint64_t operator()(const _Key& __key) const noexcept - { - if constexpr (sizeof(_Key) <= 16) - { - const _Key __copy{__key}; - return __compute_hash_span(::cuda::std::span{&__copy, 1}); - } - else - { - return __compute_hash_span(::cuda::std::span{&__key, 1}); - } - } - - //! @brief Returns a hash value for its argument, as a value of type `::cuda::std::uint64_t`. - //! - //! @tparam _Extent The extent type - //! @param __keys span of keys to hash - //! @return The resulting hash value - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint64_t - operator()(::cuda::std::span<_Key, _Extent> __keys) const noexcept - { - return __compute_hash_span(__keys); - } - -private: - //! @brief Returns a hash value for its argument, as a value of type `::cuda::std::uint64_t`. - //! - //! @tparam _Extent The extent type - //! @param __keys span of keys to hash - //! @return The resulting hash value - [[nodiscard]] _CCCL_HOST_DEVICE_API ::cuda::std::uint64_t - __compute_hash_span(::cuda::std::span __keys) const noexcept - { - auto __bytes = ::cuda::std::as_bytes(__keys).data(); - const auto __size = __keys.size_bytes(); - - size_t __offset = 0; - ::cuda::std::uint64_t __h64 = {}; - - // process data in 32-byte chunks - if (__size >= 32) - { - const auto __limit = __size - 32; - ::cuda::std::array<::cuda::std::uint64_t, 4> __v; - - __v[0] = __seed_ + __prime1 + __prime2; - __v[1] = __seed_ + __prime2; - __v[2] = __seed_; - __v[3] = __seed_ - __prime1; - - for (; __offset <= __limit; __offset += 32) - { - // pipeline 4*8byte computations - const auto __pipeline_offset = __offset / 8; - ::cuda::static_for<4>([&](auto i) { - __v[i] += ::cuda::experimental::cuco::__load_chunk<::cuda::std::uint64_t>(__bytes, __pipeline_offset + i) - * __prime2; - __v[i] = ::cuda::std::rotl(__v[i], 31); - __v[i] *= __prime1; - }); - } - - __h64 = ::cuda::std::rotl(__v[0], 1) + ::cuda::std::rotl(__v[1], 7) + ::cuda::std::rotl(__v[2], 12) - + ::cuda::std::rotl(__v[3], 18); - - ::cuda::static_for<4>([&](auto i) { - __v[i] *= __prime2; - __v[i] = ::cuda::std::rotl(__v[i], 31); - __v[i] *= __prime1; - __h64 ^= __v[i]; - __h64 = __h64 * __prime1 + __prime4; - }); - } - else - { - __h64 = __seed_ + __prime5; - } - - __h64 += __size; - - // remaining data can be processed in 8-byte chunks - if ((__size % 32) >= 8) - { - _CCCL_PRAGMA_UNROLL(4) - for (; __offset <= __size - 8; __offset += 8) - { - ::cuda::std::uint64_t __k1 = - ::cuda::experimental::cuco::__load_chunk<::cuda::std::uint64_t>(__bytes, __offset / 8) * __prime2; - __k1 = ::cuda::std::rotl(__k1, 31) * __prime1; - __h64 ^= __k1; - __h64 = ::cuda::std::rotl(__h64, 27) * __prime1 + __prime4; - } - } - - // remaining data can be processed in 4-byte chunks - if ((__size % 8) >= 4) - { - for (; __offset <= __size - 4; __offset += 4) - { - __h64 ^= (::cuda::experimental::cuco::__load_chunk<::cuda::std::uint32_t>(__bytes, __offset / 4)) * __prime1; - __h64 = ::cuda::std::rotl(__h64, 23) * __prime2 + __prime3; - } - } - - // the following loop is only needed if the size of the key is not a multiple of a previous - // block size - if (__size % 4) - { - while (__offset < __size) - { - __h64 ^= (::cuda::std::to_integer<::cuda::std::uint32_t>(__bytes[__offset])) * __prime5; - __h64 = ::cuda::std::rotl(__h64, 11) * __prime1; - ++__offset; - } - } - return __finalize(__h64); - } - - // avalanche helper - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint64_t __finalize(std::uint64_t __h) const noexcept - { - __h ^= __h >> 33; - __h *= __prime2; - __h ^= __h >> 29; - __h *= __prime3; - __h ^= __h >> 32; - return __h; - } - - ::cuda::std::uint64_t __seed_; -}; -} // namespace cuda::experimental::cuco - -#include - -#endif // _CUDAX___CUCO_DETAIL_HASH_FUNCTIONS_XXHASH_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/default_policy.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/default_policy.cuh deleted file mode 100644 index 442f3d68..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/default_policy.cuh +++ /dev/null @@ -1,126 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_HYPERLOGLOG_DEFAULT_POLICY_CUH -#define _CUDAX___CUCO_DETAIL_HYPERLOGLOG_DEFAULT_POLICY_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include - -#include -#include - -#include - -namespace cuda::experimental::cuco -{ -//! @brief Default policy for `cuda::experimental::cuco::hyperloglog`. -//! -//! Bundles the three customization points of the HLL pipeline -- hash function, bit slicing, -//! and finalizer -- into a single policy. This default reproduces the behaviour shipped by -//! `cuCollections::hyperloglog`: MSB-indexed register selection, padded leading-zero count for -//! rho, and HyperLogLog++ bias correction. Custom policies (e.g. for binary interop with -//! third-party sketch libraries) can be supplied via the `_Policy` template parameter on -//! `hyperloglog` and `hyperloglog_ref`. -//! -//! @tparam _Key The item type the sketch counts. -//! @tparam _Algo The hash algorithm. Defaults to xxhash_64. -template -struct default_hll_policy -{ - using hasher = hash<_Key, _Algo>; - using hash_result_type = decltype(::cuda::std::declval()(::cuda::std::declval<_Key>())); - using register_type = ::cuda::std::int32_t; - - static_assert(::cuda::std::is_unsigned_v, "HyperLogLog requires an unsigned hash value type"); - static_assert(::cuda::std::numeric_limits::digits == 32 - || ::cuda::std::numeric_limits::digits == 64, - "HyperLogLog requires a 32-bit or 64-bit hash value type"); - - hasher hasher_{}; - - //! @brief Returns the underlying hash functor. - //! - //! @return The hash functor. - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr hasher hash_function() const noexcept - { - return hasher_; - } - - //! @brief Hashes an item. - //! - //! @param[in] __k The item to hash. - //! @return The hash value of `__k`. - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr hash_result_type hash(const _Key& __k) const noexcept - { - return hasher_(__k); - } - - //! @brief Extracts the register index from the hash. - //! - //! @note Index is taken from the high `__precision` bits of the hash, matching Apache Spark's - //! HyperLogLog++ convention. - //! - //! @param[in] __h The hash value. - //! @param[in] __precision The HLL precision parameter. - //! @return The register index in `[0, 2^__precision)`. - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint32_t - register_index(hash_result_type __h, ::cuda::std::int32_t __precision) const noexcept - { - constexpr auto __hash_bits = ::cuda::std::numeric_limits::digits; - return static_cast<::cuda::std::uint32_t>(__h >> (__hash_bits - __precision)); - } - - //! @brief Computes rho (1 + leading zeros of the rho source) from the hash. - //! - //! @note A one-bit padding bounds the leading-zero count at `hash_bits - __precision`, - //! preventing rho overflow when the low `hash_bits - __precision` bits of the hash are zero. - //! - //! @param[in] __h The hash value. - //! @param[in] __precision The HLL precision parameter. - //! @return rho, in `[1, hash_bits - __precision + 1]`. - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint8_t - register_value(hash_result_type __h, ::cuda::std::int32_t __precision) const noexcept - { - const auto __w_padding = hash_result_type{1} << static_cast(__precision - 1); - return static_cast<::cuda::std::uint8_t>(::cuda::std::countl_zero((__h << __precision) | __w_padding) + 1); - } - - //! @brief Finalizes the GPU reduction into a cardinality estimate using the HyperLogLog++ - //! bias-corrected estimator. - //! - //! @param[in] __z Sum of `2^-register[i]` across all registers. - //! @param[in] __v Count of zero registers. - //! @param[in] __precision HLL precision parameter. - //! @return The bias-corrected cardinality estimate. - [[nodiscard]] static _CCCL_HOST_DEVICE_API constexpr double - finalize(double __z, ::cuda::std::int32_t __v, ::cuda::std::int32_t __precision) noexcept - { - return __hyperloglog_ns::hllpp_finalizer{__precision}(__z, __v); - } -}; -} // namespace cuda::experimental::cuco - -#include - -#endif // _CUDAX___CUCO_DETAIL_HYPERLOGLOG_DEFAULT_POLICY_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/finalizer.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/finalizer.cuh deleted file mode 100644 index 6e9481f9..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/finalizer.cuh +++ /dev/null @@ -1,189 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_HYPERLOGLOG_FINALIZER_CUH -#define _CUDAX___CUCO_DETAIL_HYPERLOGLOG_FINALIZER_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace cuda::experimental::cuco::__hyperloglog_ns -{ -//! @brief Estimate correction algorithm based on HyperLogLog++. -//! -//! @note Variable names correspond to the definitions given in the HLL++ paper: -//! https://static.googleusercontent.com/media/research.google.com/de//pubs/archive/40671.pdf -//! @note Precision must be >= 4. -//! -class hllpp_finalizer -{ - // Note: Most of the types in this implementation are explicit instead of relying on `auto` to - // avoid confusion with the reference implementation. - -public: - //! @brief Constructs an HLL finalizer object. - //! - //! @param __precision_ HLL precision parameter - _CCCL_HOST_DEVICE_API constexpr hllpp_finalizer(::cuda::std::int32_t __precision_) noexcept - : __precision{__precision_} - , __m{static_cast<::cuda::std::int32_t>(1u << __precision_)} - { - _CCCL_ASSERT(::cuda::in_range(__precision_, 4, 18), "Precision must be between 4 and 18"); - } - - //! @brief Compute the bias-corrected cardinality estimate. - //! - //! @param __z Geometric mean of registers - //! @param __v Number of 0 registers - //! - //! @return Bias-corrected cardinality estimate - [[nodiscard]] _CCCL_HOST_DEVICE_API double operator()(double __z, ::cuda::std::int32_t __v) const noexcept - { - double __e = __alpha_mm() / __z; - - if (__v > 0) - { - // Use linear counting for small cardinality estimates. - const double __h = __m * ::cuda::std::log(static_cast(__m) / __v); - // The threshold `2.5 * m` is from the original HLL algorithm. - if (__e <= 2.5 * __m) - { - return __h; - } - - if (__precision < 19) - { - __e = (__h <= __hyperloglog_ns::__threshold(__precision)) ? __h : __bias_corrected_estimate(__e); - } - } - else - { - // HLL++ is defined only when p < 19, otherwise we need to fallback to HLL. - if (__precision < 19) - { - __e = __bias_corrected_estimate(__e); - } - } - - return __e; - } - -private: - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr double __alpha_mm() const noexcept - { - const auto __m2 = static_cast(__m) * __m; - switch (__m) - { - case 16: - return 0.673 * __m2; - case 32: - return 0.697 * __m2; - case 64: - return 0.709 * __m2; - default: - return (0.7213 / (1.0 + 1.079 / __m)) * __m2; - } - } - - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr double __bias_corrected_estimate(double __e) const noexcept - { - return (__e < 5.0 * __m) ? __e - __bias(__e) : __e; - } - - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr double __bias(double __e) const noexcept - { - const auto __anchor_index = __interpolation_anchor_index(__e); - const auto __n = static_cast<::cuda::std::int32_t>(__hyperloglog_ns::__raw_estimate_data_size(__precision)); - - auto __low = ::cuda::std::max(__anchor_index - __k + 1, ::cuda::std::int32_t{0}); - auto __high = ::cuda::std::min(__low + __k, __n); - // Keep moving bounds as long as the (exclusive) high bound is closer to the estimate than - // the lower (inclusive) bound. - while (__high < __n && __distance(__e, __high) < __distance(__e, __low)) - { - __low += 1; - __high += 1; - } - - const auto __biases = __hyperloglog_ns::__bias_data(__precision); - double __bias_sum = 0.0; - for (::cuda::std::int32_t __i = __low; __i < __high; ++__i) - { - __bias_sum += __biases[__i]; - } - - return __bias_sum / (__high - __low); - } - - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr double __distance(double __e, ::cuda::std::int32_t __i) const noexcept - { - const auto __diff = __e - __hyperloglog_ns::__raw_estimate_data(__precision)[__i]; - return __diff * __diff; - } - - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::int32_t - __interpolation_anchor_index(double __e) const noexcept - { - const auto __estimates = __hyperloglog_ns::__raw_estimate_data(__precision); - const auto __n = static_cast<::cuda::std::int32_t>(__hyperloglog_ns::__raw_estimate_data_size(__precision)); - ::cuda::std::int32_t __left = 0; - ::cuda::std::int32_t __right = __n - 1; - - while (__left <= __right) - { - const ::cuda::std::int32_t __mid = ::cuda::std::midpoint(__left, __right); - - if (__estimates[__mid] < __e) - { - __left = __mid + 1; - } - else if (__estimates[__mid] > __e) - { - __right = __mid - 1; - } - else - { - // Exact match found, no need to look further - return __mid; - } - } - - // At this point, '__left' is the binary-search insertion point. Spark uses the insertion - // point as the anchor index when the exact estimate is not present in the table. - return __left; - } - - static constexpr ::cuda::std::int32_t __k = 6; ///< Number of interpolation points to consider - ::cuda::std::int32_t __precision; ///< HLL precision parameter - ::cuda::std::int32_t __m; ///< Number of registers (2^precision) -}; -} // namespace cuda::experimental::cuco::__hyperloglog_ns - -#include - -#endif // _CUDAX___CUCO_DETAIL_HYPERLOGLOG_FINALIZER_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/hyperloglog_impl.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/hyperloglog_impl.cuh deleted file mode 100644 index ed6dcffb..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/hyperloglog_impl.cuh +++ /dev/null @@ -1,618 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_HYPERLOGLOG_IMPL_CUH -#define _CUDAX___CUCO_DETAIL_HYPERLOGLOG_IMPL_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include - -#include -#include - -namespace cuda::experimental::cuco -{ -CUDAX_CUCO_DEFINE_STRONG_TYPE(__sketch_size_kb_t, double); - -CUDAX_CUCO_DEFINE_STRONG_TYPE(__standard_deviation_t, double); - -CUDAX_CUCO_DEFINE_STRONG_TYPE(__precision_t, ::cuda::std::int32_t); - -//! @brief A GPU-accelerated utility for approximating the number of distinct items in a multiset. -//! -//! @note This class implements the HyperLogLog/HyperLogLog++ algorithm: -//! https://static.googleusercontent.com/media/research.google.com/de//pubs/archive/40671.pdf. -//! -//! @tparam _Tp Type of items to count -//! @tparam _Scope The scope in which operations will be performed by individual threads -//! @tparam _Policy Policy bundling hash function, bit-slicing rule, and finalizer -template -class __hyperloglog_impl -{ - using __fp_type = double; ///< Floating point type used for reduction - -public: - using __value_type = _Tp; ///< Type of items to count - using __policy_type = _Policy; ///< Policy type - using __hasher = typename _Policy::hasher; ///< Hash function type - using __register_type = typename _Policy::register_type; ///< HLL register type - -private: - _Policy __policy; ///< Policy used to hash items, slice the hash, and finalize the estimate - ::cuda::std::int32_t __precision; ///< HLL precision parameter - ::cuda::std::span<__register_type> __sketch; ///< HLL sketch storage - - template - friend struct __hyperloglog_impl; - -public: - static constexpr auto __thread_scope = _Scope; ///< CUDA thread scope - - template <::cuda::thread_scope _NewScope> - using __rebind_scope = __hyperloglog_impl<_Tp, _NewScope, _Policy>; ///< Ref type with different thread scope - - //! @brief Constructs a non-owning `__hyperloglog_impl` object. - //! - //! @throw If sketch size < 0.0625KB or 64B or standard deviation > 0.2765. Throws if called from - //! host; __trap() if called from device. - //! @throw If sketch size implies precision outside [4, 18]. Throws if called from host; __trap() if - //! called from device. - //! @throw If sketch storage has insufficient alignment. Throws if called from host; __trap() if called from device. - //! - //! @param __sketch_span Reference to sketch storage - //! @param __policy The policy used to hash items and finalize the estimate - _CCCL_HOST_DEVICE_API constexpr __hyperloglog_impl(::cuda::std::span<::cuda::std::byte> __sketch_span, - const _Policy& __policy) - : __policy{__policy} - , __precision{::cuda::std::countr_zero( - __sketch_bytes(static_cast<::cuda::experimental::cuco::__sketch_size_kb_t>(__sketch_span.size() / 1024.0)) - / sizeof(__register_type))} - , __sketch{reinterpret_cast(__sketch_span.data()), __sketch_bytes() / sizeof(__register_type)} - // MSVC fails with __register_type*, use int* instead - { - constexpr ::cuda::std::size_t __minimum_sketch_bytes = sizeof(__register_type) * (1ull << 4); - if (__sketch_span.size() < __minimum_sketch_bytes) - { - _CCCL_THROW(::std::invalid_argument, "Minimum required sketch size is 0.0625KB or 64B"); - } - - if (!::cuda::is_aligned(__sketch_span.data(), __sketch_alignment())) - { - _CCCL_THROW(::std::invalid_argument, "Sketch storage has insufficient alignment"); - } - - if (!::cuda::in_range(__precision, 4, 18)) - { - _CCCL_THROW(::std::invalid_argument, "Minimum required sketch size is 0.0625KB or 64B"); - } - } - - //! @brief Resets the estimator, i.e., clears the current count estimate. - //! - //! @tparam _CG CUDA Cooperative Group type - //! - //! @param __group CUDA Cooperative group this operation is executed in - template - _CCCL_DEVICE_API constexpr void __clear(_CG __group) noexcept - { - for (int __i = __group.thread_rank(); __i < __sketch.size(); __i += __group.size()) - { - __sketch[__i] = 0; - } - } - - //! @brief Resets the estimator, i.e., clears the current count estimate. - //! - //! @note This function synchronizes the given stream. For asynchronous execution use - //! `__clear_async`. - //! - //! @param __stream CUDA stream this operation is executed in - _CCCL_HOST_API constexpr void __clear(::cuda::stream_ref __stream) - { - __clear_async(__stream); - __stream.sync(); - } - - //! @brief Asynchronously resets the estimator, i.e., clears the current count estimate. - //! - //! @param __stream CUDA stream this operation is executed in - _CCCL_HOST_API constexpr void __clear_async(::cuda::stream_ref __stream) - { - constexpr auto __block_size = 1024; - ::cuda::experimental::cuco::__hyperloglog_ns::__clear<<<1, __block_size, 0, __stream.get()>>>(*this); - } - - //! @brief Adds an item to the estimator. - //! - //! @note Hash, register index, and rho are determined by the active policy. - //! - //! @param __item The item to be counted - _CCCL_DEVICE_API constexpr void __add(const _Tp& __item) noexcept - { - const auto __h = __policy.hash(__item); - __update_max(__policy.register_index(__h, __precision), __policy.register_value(__h, __precision)); - } - - //! @brief Asynchronously adds to be counted items to the estimator. - //! - //! @tparam _InputIt Device accessible random access input iterator where - //! std::is_convertible::value_type, - //! _Tp> is `true` - //! - //! @param __first Beginning of the sequence of items - //! @param __last End of the sequence of items - //! @param __stream CUDA stream this operation is executed in - template - _CCCL_HOST_API constexpr void __add_async(_InputIt __first, _InputIt __last, ::cuda::stream_ref __stream) - { - const auto __num_items = ::cuda::std::distance(__first, __last); - if (__num_items == 0) - { - return; - } - - int __grid_size = 0; - int __block_size = 0; - const int __shmem_bytes = __sketch_bytes(); - const void* __kernel = nullptr; - - // In case the input iterator represents a contiguous memory segment we can employ efficient - // vectorized loads - if constexpr (::cuda::std::contiguous_iterator<_InputIt>) - { - const auto __ptr = ::cuda::std::to_address(__first); - constexpr auto __max_vector_bytes = 32; - const auto __alignment = - 1u << ::cuda::std::countr_zero(reinterpret_cast<::cuda::std::uintptr_t>(__ptr) | __max_vector_bytes); - const auto __vector_size = __alignment / sizeof(__value_type); - - switch (__vector_size) - { - using ::cuda::experimental::cuco::__hyperloglog_ns::__add_shmem_vectorized; - case 2: - __kernel = reinterpret_cast(__add_shmem_vectorized<2, __hyperloglog_impl>); - break; - case 4: - __kernel = reinterpret_cast(__add_shmem_vectorized<4, __hyperloglog_impl>); - break; - case 8: - __kernel = reinterpret_cast(__add_shmem_vectorized<8, __hyperloglog_impl>); - break; - case 16: - __kernel = reinterpret_cast(__add_shmem_vectorized<16, __hyperloglog_impl>); - break; - }; - } - - if (__kernel != nullptr && __try_reserve_shmem(__kernel, __shmem_bytes)) - { - if constexpr (::cuda::std::contiguous_iterator<_InputIt>) - { - // We make use of the occupancy calculator to get the minimum number of blocks which still - // saturates the GPU. This reduces the shmem initialization overhead and atomic contention - // on the final register array during the merge phase. - _CCCL_TRY_CUDA_API( - ::cudaOccupancyMaxPotentialBlockSize, - "cudaOccupancyMaxPotentialBlockSize failed", - &__grid_size, - &__block_size, - __kernel, - __shmem_bytes); - - const auto __ptr = ::cuda::std::to_address(__first); - void* __kernel_args[] = {const_cast(reinterpret_cast(&__ptr)), - const_cast(reinterpret_cast(&__num_items)), - reinterpret_cast(this)}; - _CCCL_TRY_CUDA_API( - ::cudaLaunchKernel, - "cudaLaunchKernel failed", - __kernel, - __grid_size, - __block_size, - __kernel_args, - __shmem_bytes, - __stream.get()); - } - } - else - { - __kernel = reinterpret_cast( - ::cuda::experimental::cuco::__hyperloglog_ns::__add_shmem<_InputIt, __hyperloglog_impl>); - void* __kernel_args[] = {const_cast(reinterpret_cast(&__first)), - const_cast(reinterpret_cast(&__num_items)), - reinterpret_cast(this)}; - if (__try_reserve_shmem(__kernel, __shmem_bytes)) - { - _CCCL_TRY_CUDA_API( - ::cudaOccupancyMaxPotentialBlockSize, - "cudaOccupancyMaxPotentialBlockSize failed", - &__grid_size, - &__block_size, - __kernel, - __shmem_bytes); - - _CCCL_TRY_CUDA_API( - ::cudaLaunchKernel, - "cudaLaunchKernel failed", - __kernel, - __grid_size, - __block_size, - __kernel_args, - __shmem_bytes, - __stream.get()); - } - else - { - // Computes sketch directly in global memory. (Fallback path in case there is not enough - // shared memory available) - __kernel = reinterpret_cast( - ::cuda::experimental::cuco::__hyperloglog_ns::__add_gmem<_InputIt, __hyperloglog_impl>); - - _CCCL_TRY_CUDA_API( - ::cudaOccupancyMaxPotentialBlockSize, - "cudaOccupancyMaxPotentialBlockSize failed", - &__grid_size, - &__block_size, - __kernel, - 0); - - _CCCL_TRY_CUDA_API( - ::cudaLaunchKernel, - "cudaLaunchKernel failed", - __kernel, - __grid_size, - __block_size, - __kernel_args, - 0, - __stream.get()); - } - } - } - - //! @brief Adds to be counted items to the estimator. - //! - //! @note This function synchronizes the given stream. For asynchronous execution use - //! `__add_async`. - //! - //! @tparam _InputIt Device accessible random access input iterator where - //! std::is_convertible::value_type, - //! _Tp> is `true` - //! - //! @param __first Beginning of the sequence of items - //! @param __last End of the sequence of items - //! @param __stream CUDA stream this operation is executed in - template - _CCCL_HOST_API constexpr void __add(_InputIt __first, _InputIt __last, ::cuda::stream_ref __stream) - { - __add_async(__first, __last, __stream); - __stream.sync(); - } - - //! @brief Merges the result of `other` estimator reference into `*this` estimator reference. - //! - //! @throw If __sketch_bytes() != other.__sketch_bytes(), then terminates execution with a device __trap() - //! - //! @tparam _CG CUDA Cooperative Group type - //! @tparam _OtherScope Thread scope of `other` estimator - //! - //! @param __group CUDA Cooperative group this operation is executed in - //! @param __other Other estimator reference to be merged into `*this` - template - _CCCL_DEVICE_API constexpr void __merge(_CG __group, const __hyperloglog_impl<_Tp, _OtherScope, _Policy>& __other) - { - if (__other.__precision != __precision) - { - _CCCL_THROW(::std::invalid_argument, "Cannot merge estimators with different sketch sizes"); - } - - for (int __i = __group.thread_rank(); __i < __sketch.size(); __i += __group.size()) - { - __update_max(__i, __other.__sketch[__i]); - } - } - - //! @brief Asynchronously merges the result of `other` estimator reference into `*this` - //! estimator. - //! - //! @throw If __sketch_bytes() != __other.__sketch_bytes(), then terminates execution with a device __trap() - //! - //! @tparam _OtherScope Thread scope of `other` estimator - //! - //! @param __other Other estimator reference to be merged into `*this` - //! @param __stream CUDA stream this operation is executed in - template <::cuda::thread_scope _OtherScope> - _CCCL_HOST_API constexpr void - __merge_async(const __hyperloglog_impl<_Tp, _OtherScope, _Policy>& __other, ::cuda::stream_ref __stream) - { - if (__other.__precision != __precision) - { - _CCCL_THROW(::std::invalid_argument, "Cannot merge estimators with different sketch sizes"); - } - - constexpr auto __block_size = 1024; - ::cuda::experimental::cuco::__hyperloglog_ns::__merge<<<1, __block_size, 0, __stream.get()>>>(__other, *this); - } - - //! @brief Merges the result of `other` estimator reference into `*this` estimator. - //! - //! @note This function synchronizes the given stream. For asynchronous execution use - //! `__merge_async`. - //! - //! @throw If __sketch_bytes() != __other.__sketch_bytes(), then terminates execution with a device __trap() - //! - //! @tparam _OtherScope Thread scope of `other` estimator - //! - //! @param __other Other estimator reference to be merged into `*this` - //! @param __stream CUDA stream this operation is executed in - template <::cuda::thread_scope _OtherScope> - _CCCL_HOST_API constexpr void - __merge(const __hyperloglog_impl<_Tp, _OtherScope, _Policy>& __other, ::cuda::stream_ref __stream) - { - __merge_async(__other, __stream); - __stream.sync(); - } - - //! @brief Compute the estimated distinct items count. - //! - //! @param __group CUDA thread block group this operation is executed in - //! - //! @return Approximate distinct items count - [[nodiscard]] _CCCL_DEVICE_API double __estimate(const ::cooperative_groups::thread_block& __group) const noexcept - { - __shared__ ::cuda::atomic<__fp_type, ::cuda::std::thread_scope_block> __block_sum; - __shared__ ::cuda::atomic<::cuda::std::int32_t, ::cuda::std::thread_scope_block> __block_zeroes; - __shared__ __fp_type __estimate; - - if (__group.thread_rank() == 0) - { - __block_sum.store(0); - __block_zeroes.store(0); - } - __group.sync(); - - __fp_type __thread_sum = 0; - ::cuda::std::int32_t __thread_zeroes = 0; - for (int __i = __group.thread_rank(); __i < __sketch.size(); __i += __group.size()) - { - const auto __reg = __sketch[__i]; - __thread_sum += __fp_type{1} / static_cast<__fp_type>(1ull << __reg); - __thread_zeroes += __reg == 0; - } - - // warp reduce Z and V - const auto __warp = ::cooperative_groups::tiled_partition<32, ::cooperative_groups::thread_block>(__group); - ::cooperative_groups::reduce_update_async( - __warp, __block_sum, __thread_sum, ::cooperative_groups::plus<__fp_type>()); - ::cooperative_groups::reduce_update_async( - __warp, __block_zeroes, __thread_zeroes, ::cooperative_groups::plus<::cuda::std::int32_t>()); - __group.sync(); - - if (__group.thread_rank() == 0) - { - const auto __z = __block_sum.load(::cuda::std::memory_order_relaxed); - const auto __v = __block_zeroes.load(::cuda::std::memory_order_relaxed); - __estimate = _Policy::finalize(__z, __v, __precision); - } - __group.sync(); - - return __estimate; - } - - //! @brief Compute the estimated distinct items count. - //! - //! @note This function synchronizes the given stream. - //! - //! @tparam _HostMemoryResource Host memory resource used for allocating the host buffer required to - //! compute the final estimate by copying the sketch from device to host - //! - //! @param __host_mr Host memory resource used for copying the sketch - //! @param __stream CUDA stream this operation is executed in - //! - //! @return Approximate distinct items count - template - [[nodiscard]] _CCCL_HOST_API double __estimate(_HostMemoryResource __host_mr, ::cuda::stream_ref __stream) const - { - const auto __num_regs = __sketch.size(); - - ::cuda::host_buffer<__register_type> __host_sketch_buf{__stream, __host_mr, __sketch.size(), ::cuda::no_init}; - - ::cuda::__driver::__memcpyAsync( - __host_sketch_buf.data(), __sketch.data(), sizeof(__register_type) * __num_regs, __stream.get()); - __stream.sync(); - - __fp_type __sum = 0; - ::cuda::std::int32_t __zeroes = 0; - - // geometric mean computation + count registers with 0s - for (const auto __reg : __host_sketch_buf) - { - __sum += __fp_type{1} / static_cast<__fp_type>(1ull << __reg); - __zeroes += __reg == 0; - } - - // dispatch to the policy's finalizer for bias correction, etc. - return _Policy::finalize(__sum, __zeroes, __precision); - } - - // #endif - - //! @brief Gets the hash function. - //! - //! @return The hash function, as exposed by the policy via `hash_function()`. - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __hash_function() const noexcept - { - return __policy.hash_function(); - } - - //! @brief Gets the policy. - //! - //! @return The policy - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const _Policy& __policy_() const noexcept - { - return __policy; - } - - //! @brief Gets the span of the sketch. - //! - //! @return The ::cuda::std::span of the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::span<::cuda::std::byte> __sketch_span() const noexcept - { - return ::cuda::std::span<::cuda::std::byte>(reinterpret_cast<::cuda::std::byte*>(__sketch.data()), __sketch_bytes()); - } - - //! @brief Gets the number of bytes required for the sketch storage. - //! - //! @return The number of bytes required for the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::size_t __sketch_bytes() const noexcept - { - return (1ull << __precision) * sizeof(__register_type); - } - - //! @brief Gets the number of bytes required for the sketch storage. - //! - //! @param sketch_size_kb Upper bound sketch size in KB - //! - //! @return The number of bytes required for the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr ::cuda::std::size_t - __sketch_bytes(::cuda::experimental::cuco::__sketch_size_kb_t __sketch_size_kb) noexcept - { - // minimum precision is 4 or 64 bytes - return ::cuda::std::max(static_cast<::cuda::std::size_t>(sizeof(__register_type) * (1ull << 4)), - ::cuda::std::bit_floor(static_cast<::cuda::std::size_t>(__sketch_size_kb * 1024))); - } - - //! @brief Gets the number of bytes required for the sketch storage. - //! - //! @param __standard_deviation Upper bound standard deviation for approximation error - //! - //! @return The number of bytes required for the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr ::cuda::std::size_t - sketch_bytes(::cuda::experimental::cuco::__standard_deviation_t __standard_deviation) noexcept - { - // implementation taken from - // https://github.com/apache/spark/blob/6a27789ad7d59cd133653a49be0bb49729542abe/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/HyperLogLogPlusPlusHelper.scala#L43 - - const auto __precision_from_sd = - static_cast<::cuda::std::int32_t>(::cuda::std::ceil(2.0 * ::cuda::std::log2(1.106 / __standard_deviation))); - - // minimum precision is 4 or 64 bytes - const auto __precision_ = ::cuda::std::max(::cuda::std::int32_t{4}, __precision_from_sd); - - // inverse of this function (omitting the minimum precision constraint) is - // standard_deviation = 1.106 / exp((__precision_ * log(2.0)) / 2.0) - - return sizeof(__register_type) * (1ull << __precision_); - } - - //! @brief Gets the number of bytes required for the sketch storage. - //! - //! @param __precision HyperLogLog precision parameter - //! - //! @return The number of bytes required for the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr ::cuda::std::size_t - sketch_bytes(::cuda::experimental::cuco::__precision_t __precision) noexcept - { - const auto __precision_value = static_cast<::cuda::std::int32_t>(__precision); - - return sizeof(__register_type) * (1ull << __precision_value); - } - - //! @brief Gets the alignment required for the sketch storage. - //! - //! @return The required alignment - [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr ::cuda::std::size_t __sketch_alignment() noexcept - { - return alignof(__register_type); - } - -private: - //! @brief Atomically updates the register at position `i` with `max(reg[i], value)`. - //! - //! @param __i Register index - //! @param __value New value - _CCCL_DEVICE_API constexpr void __update_max(int __i, __register_type __value) noexcept - { - ::cuda::atomic_ref<__register_type, _Scope> __register_ref(__sketch[__i]); - __register_ref.fetch_max(__value, ::cuda::memory_order_relaxed); - } - - //! @brief Try expanding the shmem partition for a given kernel beyond 48KB if necessary. - //! - //! @tparam _Kernel Type of kernel function - //! - //! @param __kernel The kernel function - //! @param __shmem_bytes Number of requested dynamic shared memory bytes - //! - //! @returns True iff kernel configuration is successful - template - [[nodiscard]] _CCCL_HOST_API constexpr bool __try_reserve_shmem(_Kernel __kernel, int __shmem_bytes) const - { - int __device = -1; - _CCCL_TRY_CUDA_API(::cudaGetDevice, "cudaGetDevice failed", &__device); - int __max_shmem_bytes = 0; - _CCCL_TRY_CUDA_API( - ::cudaDeviceGetAttribute, - "cudaDeviceGetAttribute failed", - &__max_shmem_bytes, - ::cudaDevAttrMaxSharedMemoryPerBlockOptin, - __device); - - if (__shmem_bytes <= __max_shmem_bytes) - { - _CCCL_TRY_CUDA_API( - ::cudaFuncSetAttribute, - "cudaFuncSetAttribute failed", - reinterpret_cast(__kernel), - cudaFuncAttributeMaxDynamicSharedMemorySize, - __shmem_bytes); - return true; - } - else - { - return false; - } - } -}; -} // namespace cuda::experimental::cuco - -#include - -#endif // _CUDAX___CUCO_DETAIL_HYPERLOGLOG_IMPL_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/kernels.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/kernels.cuh deleted file mode 100644 index a33e2f47..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/kernels.cuh +++ /dev/null @@ -1,182 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_HYPERLOGLOG_KERNELS_CUH -#define _CUDAX___CUCO_DETAIL_HYPERLOGLOG_KERNELS_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include - -#include - -#include - -#if _CCCL_CUDA_COMPILATION() - -_CCCL_DIAG_PUSH -_CCCL_DIAG_SUPPRESS_GCC("-Wattributes") - -namespace cuda::experimental::cuco::__hyperloglog_ns -{ -//! @brief Returns the global thread ID in a 1D grid -//! -//! @return The global thread ID -[[nodiscard]] _CCCL_DEVICE_API inline ::cuda::std::int64_t __global_thread_id() noexcept -{ - return static_cast<::cuda::std::int64_t>(blockDim.x) * blockIdx.x + threadIdx.x; -} - -//! @brief Returns the grid stride of a 1D grid -//! -//! @return The grid stride -[[nodiscard]] _CCCL_DEVICE_API inline ::cuda::std::int64_t __grid_stride() noexcept -{ - return static_cast<::cuda::std::int64_t>(gridDim.x) * blockDim.x; -} - -template -_CCCL_KERNEL_ATTRIBUTES void __clear(_RefType __ref) -{ - const auto __block = ::cooperative_groups::this_thread_block(); - if (__block.group_index().x == 0) - { - __ref.__clear(__block); - } -} - -template -_CCCL_KERNEL_ATTRIBUTES void -__add_shmem_vectorized(const typename _RefType::__value_type* __first, ::cuda::std::int64_t __n, _RefType __ref) -{ - using __value_type = typename _RefType::__value_type; - // TODO: replace with ::cuda::__vector_type - using __vector_type = ::cuda::std::array<__value_type, _VectorSize>; - using __local_ref_type = typename _RefType::template __rebind_scope<::cuda::std::thread_scope_block>; - - // Base address of dynamic shared memory is guaranteed to be aligned to at least 16 bytes which is - // sufficient for this purpose - extern __shared__ ::cuda::std::byte __local_sketch[]; - - const auto __loop_stride = __grid_stride(); - auto __idx = __global_thread_id(); - const auto __grid = ::cooperative_groups::this_grid(); - const auto __block = ::cooperative_groups::this_thread_block(); - - __local_ref_type __local_ref(::cuda::std::span{__local_sketch, __ref.__sketch_bytes()}, {}); - __local_ref.__clear(__block); - __block.sync(); - - // each thread processes VectorSize-many items per iteration - __vector_type __vec; - while (__idx < __n / _VectorSize) - { - __vec = *reinterpret_cast( - ::cuda::std::assume_aligned(__first + __idx * _VectorSize)); - for (int i = 0; i < _VectorSize; ++i) - { - __local_ref.__add(__vec[i]); - } - - __idx += __loop_stride; - } - // a single thread processes the remaining items -# if _CCCL_CTK_AT_LEAST(12, 1) - ::cooperative_groups::invoke_one(__grid, [&]() { - const auto __remainder = __n % _VectorSize; - for (int __i = 0; __i < __remainder; ++__i) - { - __local_ref.__add(*(__first + __n - __i - 1)); - } - }); -# else // ^^^ _CCCL_CTK_AT_LEAST(12, 1) ^^^ / vvv _CCCL_CTK_BELOW(12, 1) vvv - if (__grid.thread_rank() == 0) - { - const auto __remainder = __n % _VectorSize; - for (int __i = 0; __i < __remainder; ++__i) - { - __local_ref.__add(*(__first + __n - __i - 1)); - } - } -# endif // ^^^ _CCCL_CTK_BELOW(12, 1) ^^^ - - __block.sync(); - - __ref.__merge(__block, __local_ref); -} - -template -_CCCL_KERNEL_ATTRIBUTES void __add_shmem(_InputIt __first, ::cuda::std::int64_t __n, _RefType __ref) -{ - using __local_ref_type = typename _RefType::template __rebind_scope<::cuda::std::thread_scope_block>; - - // TODO assert alignment - extern __shared__ ::cuda::std::byte __local_sketch[]; - - const auto __loop_stride = __grid_stride(); - auto __idx = __global_thread_id(); - const auto __block = ::cooperative_groups::this_thread_block(); - - __local_ref_type __local_ref(::cuda::std::span{__local_sketch, __ref.__sketch_bytes()}, {}); - __local_ref.__clear(__block); - __block.sync(); - - while (__idx < __n) - { - __local_ref.__add(*(__first + __idx)); - __idx += __loop_stride; - } - __block.sync(); - - __ref.__merge(__block, __local_ref); -} - -template -_CCCL_KERNEL_ATTRIBUTES void __add_gmem(_InputIt __first, ::cuda::std::int64_t __n, _RefType __ref) -{ - const auto __loop_stride = __grid_stride(); - auto __idx = __global_thread_id(); - - while (__idx < __n) - { - __ref.__add(*(__first + __idx)); - __idx += __loop_stride; - } -} - -template -_CCCL_KERNEL_ATTRIBUTES void __merge(_OtherRefType __other_ref, _RefType __ref) -{ - const auto __block = ::cooperative_groups::this_thread_block(); - if (__block.group_index().x == 0) - { - __ref.__merge(__block, __other_ref); - } -} -} // namespace cuda::experimental::cuco::__hyperloglog_ns - -_CCCL_DIAG_POP - -#endif // _CCCL_CUDA_COMPILATION() - -#include - -#endif // _CUDAX___CUCO_DETAIL_HYPERLOGLOG_KERNELS_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/tuning.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/tuning.cuh deleted file mode 100644 index 5f8dc329..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/hyperloglog/tuning.cuh +++ /dev/null @@ -1,166 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_HYPERLOGLOG_TUNING_CUH -#define _CUDAX___CUCO_DETAIL_HYPERLOGLOG_TUNING_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include - -#include - -namespace cuda::experimental::cuco::__hyperloglog_ns -{ -#ifndef _CUDAX_CUCO_HLL_TUNING_ARR_DECL -# if _CCCL_OS(WINDOWS) -# define _CUDAX_CUCO_HLL_TUNING_ARR_DECL _CCCL_GLOBAL_CONSTANT double -# else -# define _CUDAX_CUCO_HLL_TUNING_ARR_DECL _CCCL_DEVICE inline constexpr double -# endif -#endif - -// clang-format off -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __threshold_data[] = {10.0, 20.0, 40.0, 80.0, 220.0, 400.0, 900.0, 1800.0, 3100.0, 6500.0, 15500.0, 20000.0, 50000.0, 120000.0, 350000.0}; - -//! @brief Get threshold value for a given precision -//! -//! @param __precision The precision value (4-18) -//! @return The threshold value for the given precision -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __threshold(::cuda::std::int32_t __precision) noexcept { - return __threshold_data[__precision - 4]; -} - -// HLL++ uses an interpolation method over the raw estimated cardinality to select the optimal bias. -// Parameters/interpolation points taken from -// https://docs.google.com/document/d/1gyjfMHy43U9OWBXxfaeG-3MjGzejW1dlpyMwEYAAWEI/mobilebasic -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p4[] = {11.0, 11.717, 12.207, 12.7896, 13.2882, 13.8204, 14.3772, 14.9342, 15.5202, 16.161, 16.7722, 17.4636, 18.0396, 18.6766, 19.3566, 20.0454, 20.7936, 21.4856, 22.2666, 22.9946, 23.766, 24.4692, 25.3638, 26.0764, 26.7864, 27.7602, 28.4814, 29.433, 30.2926, 31.0664, 31.9996, 32.7956, 33.5366, 34.5894, 35.5738, 36.2698, 37.3682, 38.0544, 39.2342, 40.0108, 40.7966, 41.9298, 42.8704, 43.6358, 44.5194, 45.773, 46.6772, 47.6174, 48.4888, 49.3304, 50.2506, 51.4996, 52.3824, 53.3078, 54.3984, 55.5838, 56.6618, 57.2174, 58.3514, 59.0802, 60.1482, 61.0376, 62.3598, 62.8078, 63.9744, 64.914, 65.781, 67.1806, 68.0594, 68.8446, 69.7928, 70.8248, 71.8324, 72.8598, 73.6246, 74.7014, 75.393, 76.6708, 77.2394}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p5[] = {23.0, 23.1194, 23.8208, 24.2318, 24.77, 25.2436, 25.7774, 26.2848, 26.8224, 27.3742, 27.9336, 28.503, 29.0494, 29.6292, 30.2124, 30.798, 31.367, 31.9728, 32.5944, 33.217, 33.8438, 34.3696, 35.0956, 35.7044, 36.324, 37.0668, 37.6698, 38.3644, 39.049, 39.6918, 40.4146, 41.082, 41.687, 42.5398, 43.2462, 43.857, 44.6606, 45.4168, 46.1248, 46.9222, 47.6804, 48.447, 49.3454, 49.9594, 50.7636, 51.5776, 52.331, 53.19, 53.9676, 54.7564, 55.5314, 56.4442, 57.3708, 57.9774, 58.9624, 59.8796, 60.755, 61.472, 62.2076, 63.1024, 63.8908, 64.7338, 65.7728, 66.629, 67.413, 68.3266, 69.1524, 70.2642, 71.1806, 72.0566, 72.9192, 73.7598, 74.3516, 75.5802, 76.4386, 77.4916, 78.1524, 79.1892, 79.8414, 80.8798, 81.8376, 82.4698, 83.7656, 84.331, 85.5914, 86.6012, 87.7016, 88.5582, 89.3394, 90.3544, 91.4912, 92.308, 93.3552, 93.9746, 95.2052, 95.727, 97.1322, 98.3944, 98.7588, 100.242, 101.1914, 102.2538, 102.8776, 103.6292, 105.1932, 105.9152, 107.0868, 107.6728, 108.7144, 110.3114, 110.8716, 111.245, 112.7908, 113.7064, 114.636, 115.7464, 116.1788, 117.7464, 118.4896, 119.6166, 120.5082, 121.7798, 122.9028, 123.4426, 124.8854, 125.705, 126.4652, 128.3464, 128.3462, 130.0398, 131.0342, 131.0042, 132.4766, 133.511, 134.7252, 135.425, 136.5172, 138.0572, 138.6694, 139.3712, 140.8598, 141.4594, 142.554, 143.4006, 144.7374, 146.1634, 146.8994, 147.605, 147.9304, 149.1636, 150.2468, 151.5876, 152.2096, 153.7032, 154.7146, 155.807, 156.9228, 157.0372, 158.5852}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p6[] = {46.0, 46.1902, 47.271, 47.8358, 48.8142, 49.2854, 50.317, 51.354, 51.8924, 52.9436, 53.4596, 54.5262, 55.6248, 56.1574, 57.2822, 57.837, 58.9636, 60.074, 60.7042, 61.7976, 62.4772, 63.6564, 64.7942, 65.5004, 66.686, 67.291, 68.5672, 69.8556, 70.4982, 71.8204, 72.4252, 73.7744, 75.0786, 75.8344, 77.0294, 77.8098, 79.0794, 80.5732, 81.1878, 82.5648, 83.2902, 84.6784, 85.3352, 86.8946, 88.3712, 89.0852, 90.499, 91.2686, 92.6844, 94.2234, 94.9732, 96.3356, 97.2286, 98.7262, 100.3284, 101.1048, 102.5962, 103.3562, 105.1272, 106.4184, 107.4974, 109.0822, 109.856, 111.48, 113.2834, 114.0208, 115.637, 116.5174, 118.0576, 119.7476, 120.427, 122.1326, 123.2372, 125.2788, 126.6776, 127.7926, 129.1952, 129.9564, 131.6454, 133.87, 134.5428, 136.2, 137.0294, 138.6278, 139.6782, 141.792, 143.3516, 144.2832, 146.0394, 147.0748, 148.4912, 150.849, 151.696, 153.5404, 154.073, 156.3714, 157.7216, 158.7328, 160.4208, 161.4184, 163.9424, 165.2772, 166.411, 168.1308, 168.769, 170.9258, 172.6828, 173.7502, 175.706, 176.3886, 179.0186, 180.4518, 181.927, 183.4172, 184.4114, 186.033, 188.5124, 189.5564, 191.6008, 192.4172, 193.8044, 194.997, 197.4548, 198.8948, 200.2346, 202.3086, 203.1548, 204.8842, 206.6508, 206.6772, 209.7254, 210.4752, 212.7228, 214.6614, 215.1676, 217.793, 218.0006, 219.9052, 221.66, 223.5588, 225.1636, 225.6882, 227.7126, 229.4502, 231.1978, 232.9756, 233.1654, 236.727, 238.1974, 237.7474, 241.1346, 242.3048, 244.1948, 245.3134, 246.879, 249.1204, 249.853, 252.6792, 253.857, 254.4486, 257.2362, 257.9534, 260.0286, 260.5632, 262.663, 264.723, 265.7566, 267.2566, 267.1624, 270.62, 272.8216, 273.2166, 275.2056, 276.2202, 278.3726, 280.3344, 281.9284, 283.9728, 284.1924, 286.4872, 287.587, 289.807, 291.1206, 292.769, 294.8708, 296.665, 297.1182, 299.4012, 300.6352, 302.1354, 304.1756, 306.1606, 307.3462, 308.5214, 309.4134, 310.8352, 313.9684, 315.837, 316.7796, 318.9858}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p7[] = {92.0, 93.4934, 94.9758, 96.4574, 97.9718, 99.4954, 101.5302, 103.0756, 104.6374, 106.1782, 107.7888, 109.9522, 111.592, 113.2532, 114.9086, 116.5938, 118.9474, 120.6796, 122.4394, 124.2176, 125.9768, 128.4214, 130.2528, 132.0102, 133.8658, 135.7278, 138.3044, 140.1316, 142.093, 144.0032, 145.9092, 148.6306, 150.5294, 152.5756, 154.6508, 156.662, 159.552, 161.3724, 163.617, 165.5754, 167.7872, 169.8444, 172.7988, 174.8606, 177.2118, 179.3566, 181.4476, 184.5882, 186.6816, 189.0824, 191.0258, 193.6048, 196.4436, 198.7274, 200.957, 203.147, 205.4364, 208.7592, 211.3386, 213.781, 215.8028, 218.656, 221.6544, 223.996, 226.4718, 229.1544, 231.6098, 234.5956, 237.0616, 239.5758, 242.4878, 244.5244, 248.2146, 250.724, 252.8722, 255.5198, 258.0414, 261.941, 264.9048, 266.87, 269.4304, 272.028, 274.4708, 278.37, 281.0624, 283.4668, 286.5532, 289.4352, 293.2564, 295.2744, 298.2118, 300.7472, 304.1456, 307.2928, 309.7504, 312.5528, 315.979, 318.2102, 322.1834, 324.3494, 327.325, 330.6614, 332.903, 337.2544, 339.9042, 343.215, 345.2864, 348.0814, 352.6764, 355.301, 357.139, 360.658, 363.1732, 366.5902, 369.9538, 373.0828, 375.922, 378.9902, 382.7328, 386.4538, 388.1136, 391.2234, 394.0878, 396.708, 401.1556, 404.1852, 406.6372, 409.6822, 412.7796, 416.6078, 418.4916, 422.131, 424.5376, 428.1988, 432.211, 434.4502, 438.5282, 440.912, 444.0448, 447.7432, 450.8524, 453.7988, 456.7858, 458.8868, 463.9886, 466.5064, 468.9124, 472.6616, 475.4682, 478.582, 481.304, 485.2738, 488.6894, 490.329, 496.106, 497.6908, 501.1374, 504.5322, 506.8848, 510.3324, 513.4512, 516.179, 520.4412, 522.6066, 526.167, 528.7794, 533.379, 536.067, 538.46, 542.9116, 545.692, 547.9546, 552.493, 555.2722, 557.335, 562.449, 564.2014, 569.0738, 571.0974, 574.8564, 578.2996, 581.409, 583.9704, 585.8098, 589.6528, 594.5998, 595.958, 600.068, 603.3278, 608.2016, 609.9632, 612.864, 615.43, 620.7794, 621.272, 625.8644, 629.206, 633.219, 634.5154, 638.6102}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p8[] = {184.2152, 187.2454, 190.2096, 193.6652, 196.6312, 199.6822, 203.249, 206.3296, 210.0038, 213.2074, 216.4612, 220.27, 223.5178, 227.4412, 230.8032, 234.1634, 238.1688, 241.6074, 245.6946, 249.2664, 252.8228, 257.0432, 260.6824, 264.9464, 268.6268, 272.2626, 276.8376, 280.4034, 284.8956, 288.8522, 292.7638, 297.3552, 301.3556, 305.7526, 309.9292, 313.8954, 318.8198, 322.7668, 327.298, 331.6688, 335.9466, 340.9746, 345.1672, 349.3474, 354.3028, 358.8912, 364.114, 368.4646, 372.9744, 378.4092, 382.6022, 387.843, 392.5684, 397.1652, 402.5426, 407.4152, 412.5388, 417.3592, 422.1366, 427.486, 432.3918, 437.5076, 442.509, 447.3834, 453.3498, 458.0668, 463.7346, 469.1228, 473.4528, 479.7, 484.644, 491.0518, 495.5774, 500.9068, 506.432, 512.1666, 517.434, 522.6644, 527.4894, 533.6312, 538.3804, 544.292, 550.5496, 556.0234, 562.8206, 566.6146, 572.4188, 579.117, 583.6762, 590.6576, 595.7864, 601.509, 607.5334, 612.9204, 619.772, 624.2924, 630.8654, 636.1836, 642.745, 649.1316, 655.0386, 660.0136, 666.6342, 671.6196, 678.1866, 684.4282, 689.3324, 695.4794, 702.5038, 708.129, 713.528, 720.3204, 726.463, 732.7928, 739.123, 744.7418, 751.2192, 756.5102, 762.6066, 769.0184, 775.2224, 781.4014, 787.7618, 794.1436, 798.6506, 805.6378, 811.766, 819.7514, 824.5776, 828.7322, 837.8048, 843.6302, 849.9336, 854.4798, 861.3388, 867.9894, 873.8196, 880.3136, 886.2308, 892.4588, 899.0816, 905.4076, 912.0064, 917.3878, 923.619, 929.998, 937.3482, 943.9506, 947.991, 955.1144, 962.203, 968.8222, 975.7324, 981.7826, 988.7666, 994.2648, 1000.3128, 1007.4082, 1013.7536, 1020.3376, 1026.7156, 1031.7478, 1037.4292, 1045.393, 1051.2278, 1058.3434, 1062.8726, 1071.884, 1076.806, 1082.9176, 1089.1678, 1095.5032, 1102.525, 1107.2264, 1115.315, 1120.93, 1127.252, 1134.1496, 1139.0408, 1147.5448, 1153.3296, 1158.1974, 1166.5262, 1174.3328, 1175.657, 1184.4222, 1190.9172, 1197.1292, 1204.4606, 1210.4578, 1218.8728, 1225.3336, 1226.6592, 1236.5768, 1241.363, 1249.4074, 1254.6566, 1260.8014, 1266.5454, 1274.5192}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p9[] = {369.0, 374.8294, 381.2452, 387.6698, 394.1464, 400.2024, 406.8782, 413.6598, 420.462, 427.2826, 433.7102, 440.7416, 447.9366, 455.1046, 462.285, 469.0668, 476.306, 483.8448, 491.301, 498.9886, 506.2422, 513.8138, 521.7074, 529.7428, 537.8402, 545.1664, 553.3534, 561.594, 569.6886, 577.7876, 585.65, 594.228, 602.8036, 611.1666, 620.0818, 628.0824, 637.2574, 646.302, 655.1644, 664.0056, 672.3802, 681.7192, 690.5234, 700.2084, 708.831, 718.485, 728.1112, 737.4764, 746.76, 756.3368, 766.5538, 775.5058, 785.2646, 795.5902, 804.3818, 814.8998, 824.9532, 835.2062, 845.2798, 854.4728, 864.9582, 875.3292, 886.171, 896.781, 906.5716, 916.7048, 927.5322, 937.875, 949.3972, 958.3464, 969.7274, 980.2834, 992.1444, 1003.4264, 1013.0166, 1024.018, 1035.0438, 1046.34, 1057.6856, 1068.9836, 1079.0312, 1091.677, 1102.3188, 1113.4846, 1124.4424, 1135.739, 1147.1488, 1158.9202, 1169.406, 1181.5342, 1193.2834, 1203.8954, 1216.3286, 1226.2146, 1239.6684, 1251.9946, 1262.123, 1275.4338, 1285.7378, 1296.076, 1308.9692, 1320.4964, 1333.0998, 1343.9864, 1357.7754, 1368.3208, 1380.4838, 1392.7388, 1406.0758, 1416.9098, 1428.9728, 1440.9228, 1453.9292, 1462.617, 1476.05, 1490.2996, 1500.6128, 1513.7392, 1524.5174, 1536.6322, 1548.2584, 1562.3766, 1572.423, 1587.1232, 1596.5164, 1610.5938, 1622.5972, 1633.1222, 1647.7674, 1658.5044, 1671.57, 1683.7044, 1695.4142, 1708.7102, 1720.6094, 1732.6522, 1747.841, 1756.4072, 1769.9786, 1782.3276, 1797.5216, 1808.3186, 1819.0694, 1834.354, 1844.575, 1856.2808, 1871.1288, 1880.7852, 1893.9622, 1906.3418, 1920.6548, 1932.9302, 1945.8584, 1955.473, 1968.8248, 1980.6446, 1995.9598, 2008.349, 2019.8556, 2033.0334, 2044.0206, 2059.3956, 2069.9174, 2082.6084, 2093.7036, 2106.6108, 2118.9124, 2132.301, 2144.7628, 2159.8422, 2171.0212, 2183.101, 2193.5112, 2208.052, 2221.3194, 2233.3282, 2247.295, 2257.7222, 2273.342, 2286.5638, 2299.6786, 2310.8114, 2322.3312, 2335.516, 2349.874, 2363.5968, 2373.865, 2387.1918, 2401.8328, 2414.8496, 2424.544, 2436.7592, 2447.1682, 2464.1958, 2474.3438, 2489.0006, 2497.4526, 2513.6586, 2527.19, 2540.7028, 2553.768}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p10[] = {738.1256, 750.4234, 763.1064, 775.4732, 788.4636, 801.0644, 814.488, 827.9654, 841.0832, 854.7864, 868.1992, 882.2176, 896.5228, 910.1716, 924.7752, 938.899, 953.6126, 968.6492, 982.9474, 998.5214, 1013.1064, 1028.6364, 1044.2468, 1059.4588, 1075.3832, 1091.0584, 1106.8606, 1123.3868, 1139.5062, 1156.1862, 1172.463, 1189.339, 1206.1936, 1223.1292, 1240.1854, 1257.2908, 1275.3324, 1292.8518, 1310.5204, 1328.4854, 1345.9318, 1364.552, 1381.4658, 1400.4256, 1419.849, 1438.152, 1456.8956, 1474.8792, 1494.118, 1513.62, 1532.5132, 1551.9322, 1570.7726, 1590.6086, 1610.5332, 1630.5918, 1650.4294, 1669.7662, 1690.4106, 1710.7338, 1730.9012, 1750.4486, 1770.1556, 1791.6338, 1812.7312, 1833.6264, 1853.9526, 1874.8742, 1896.8326, 1918.1966, 1939.5594, 1961.07, 1983.037, 2003.1804, 2026.071, 2047.4884, 2070.0848, 2091.2944, 2114.333, 2135.9626, 2158.2902, 2181.0814, 2202.0334, 2224.4832, 2246.39, 2269.7202, 2292.1714, 2314.2358, 2338.9346, 2360.891, 2384.0264, 2408.3834, 2430.1544, 2454.8684, 2476.9896, 2501.4368, 2522.8702, 2548.0408, 2570.6738, 2593.5208, 2617.0158, 2640.2302, 2664.0962, 2687.4986, 2714.2588, 2735.3914, 2759.6244, 2781.8378, 2808.0072, 2830.6516, 2856.2454, 2877.2136, 2903.4546, 2926.785, 2951.2294, 2976.468, 3000.867, 3023.6508, 3049.91, 3073.5984, 3098.162, 3121.5564, 3146.2328, 3170.9484, 3195.5902, 3221.3346, 3242.7032, 3271.6112, 3296.5546, 3317.7376, 3345.072, 3369.9518, 3394.326, 3418.1818, 3444.6926, 3469.086, 3494.2754, 3517.8698, 3544.248, 3565.3768, 3588.7234, 3616.979, 3643.7504, 3668.6812, 3695.72, 3719.7392, 3742.6224, 3770.4456, 3795.6602, 3819.9058, 3844.002, 3869.517, 3895.6824, 3920.8622, 3947.1364, 3973.985, 3995.4772, 4021.62, 4046.628, 4074.65, 4096.2256, 4121.831, 4146.6406, 4173.276, 4195.0744, 4223.9696, 4251.3708, 4272.9966, 4300.8046, 4326.302, 4353.1248, 4374.312, 4403.0322, 4426.819, 4450.0598, 4478.5206, 4504.8116, 4528.8928, 4553.9584, 4578.8712, 4603.8384, 4632.3872, 4655.5128, 4675.821, 4704.6222, 4731.9862, 4755.4174, 4781.2628, 4804.332, 4832.3048, 4862.8752, 4883.4148, 4906.9544, 4935.3516, 4954.3532, 4984.0248, 5011.217, 5035.3258, 5057.3672, 5084.1828}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p11[] = {1477.0, 1501.6014, 1526.5802, 1551.7942, 1577.3042, 1603.2062, 1629.8402, 1656.2292, 1682.9462, 1709.9926, 1737.3026, 1765.4252, 1793.0578, 1821.6092, 1849.626, 1878.5568, 1908.527, 1937.5154, 1967.1874, 1997.3878, 2027.37, 2058.1972, 2089.5728, 2120.1012, 2151.9668, 2183.292, 2216.0772, 2247.8578, 2280.6562, 2313.041, 2345.714, 2380.3112, 2414.1806, 2447.9854, 2481.656, 2516.346, 2551.5154, 2586.8378, 2621.7448, 2656.6722, 2693.5722, 2729.1462, 2765.4124, 2802.8728, 2838.898, 2876.408, 2913.4926, 2951.4938, 2989.6776, 3026.282, 3065.7704, 3104.1012, 3143.7388, 3181.6876, 3221.1872, 3261.5048, 3300.0214, 3339.806, 3381.409, 3421.4144, 3461.4294, 3502.2286, 3544.651, 3586.6156, 3627.337, 3670.083, 3711.1538, 3753.5094, 3797.01, 3838.6686, 3882.1678, 3922.8116, 3967.9978, 4009.9204, 4054.3286, 4097.5706, 4140.6014, 4185.544, 4229.5976, 4274.583, 4316.9438, 4361.672, 4406.2786, 4451.8628, 4496.1834, 4543.505, 4589.1816, 4632.5188, 4678.2294, 4724.8908, 4769.0194, 4817.052, 4861.4588, 4910.1596, 4956.4344, 5002.5238, 5048.13, 5093.6374, 5142.8162, 5187.7894, 5237.3984, 5285.6078, 5331.0858, 5379.1036, 5428.6258, 5474.6018, 5522.7618, 5571.5822, 5618.59, 5667.9992, 5714.88, 5763.454, 5808.6982, 5860.3644, 5910.2914, 5953.571, 6005.9232, 6055.1914, 6104.5882, 6154.5702, 6199.7036, 6251.1764, 6298.7596, 6350.0302, 6398.061, 6448.4694, 6495.933, 6548.0474, 6597.7166, 6646.9416, 6695.9208, 6742.6328, 6793.5276, 6842.1934, 6894.2372, 6945.3864, 6996.9228, 7044.2372, 7094.1374, 7142.2272, 7192.2942, 7238.8338, 7288.9006, 7344.0908, 7394.8544, 7443.5176, 7490.4148, 7542.9314, 7595.6738, 7641.9878, 7694.3688, 7743.0448, 7797.522, 7845.53, 7899.594, 7950.3132, 7996.455, 8050.9442, 8092.9114, 8153.1374, 8197.4472, 8252.8278, 8301.8728, 8348.6776, 8401.4698, 8453.551, 8504.6598, 8553.8944, 8604.1276, 8657.6514, 8710.3062, 8758.908, 8807.8706, 8862.1702, 8910.4668, 8960.77, 9007.2766, 9063.164, 9121.0534, 9164.1354, 9218.1594, 9267.767, 9319.0594, 9372.155, 9419.7126, 9474.3722, 9520.1338, 9572.368, 9622.7702, 9675.8448, 9726.5396, 9778.7378, 9827.6554, 9878.1922, 9928.7782, 9978.3984, 10026.578, 10076.5626, 10137.1618, 10177.5244, 10229.9176}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p12[] = {2954.0, 3003.4782, 3053.3568, 3104.3666, 3155.324, 3206.9598, 3259.648, 3312.539, 3366.1474, 3420.2576, 3474.8376, 3530.6076, 3586.451, 3643.38, 3700.4104, 3757.5638, 3815.9676, 3875.193, 3934.838, 3994.8548, 4055.018, 4117.1742, 4178.4482, 4241.1294, 4304.4776, 4367.4044, 4431.8724, 4496.3732, 4561.4304, 4627.5326, 4693.949, 4761.5532, 4828.7256, 4897.6182, 4965.5186, 5034.4528, 5104.865, 5174.7164, 5244.6828, 5316.6708, 5387.8312, 5459.9036, 5532.476, 5604.8652, 5679.6718, 5753.757, 5830.2072, 5905.2828, 5980.0434, 6056.6264, 6134.3192, 6211.5746, 6290.0816, 6367.1176, 6447.9796, 6526.5576, 6606.1858, 6686.9144, 6766.1142, 6847.0818, 6927.9664, 7010.9096, 7091.0816, 7175.3962, 7260.3454, 7344.018, 7426.4214, 7511.3106, 7596.0686, 7679.8094, 7765.818, 7852.4248, 7936.834, 8022.363, 8109.5066, 8200.4554, 8288.5832, 8373.366, 8463.4808, 8549.7682, 8642.0522, 8728.3288, 8820.9528, 8907.727, 9001.0794, 9091.2522, 9179.988, 9269.852, 9362.6394, 9453.642, 9546.9024, 9640.6616, 9732.6622, 9824.3254, 9917.7484, 10007.9392, 10106.7508, 10196.2152, 10289.8114, 10383.5494, 10482.3064, 10576.8734, 10668.7872, 10764.7156, 10862.0196, 10952.793, 11049.9748, 11146.0702, 11241.4492, 11339.2772, 11434.2336, 11530.741, 11627.6136, 11726.311, 11821.5964, 11918.837, 12015.3724, 12113.0162, 12213.0424, 12306.9804, 12408.4518, 12504.8968, 12604.586, 12700.9332, 12798.705, 12898.5142, 12997.0488, 13094.788, 13198.475, 13292.7764, 13392.9698, 13486.8574, 13590.1616, 13686.5838, 13783.6264, 13887.2638, 13992.0978, 14081.0844, 14189.9956, 14280.0912, 14382.4956, 14486.4384, 14588.1082, 14686.2392, 14782.276, 14888.0284, 14985.1864, 15088.8596, 15187.0998, 15285.027, 15383.6694, 15495.8266, 15591.3736, 15694.2008, 15790.3246, 15898.4116, 15997.4522, 16095.5014, 16198.8514, 16291.7492, 16402.6424, 16499.1266, 16606.2436, 16697.7186, 16796.3946, 16902.3376, 17005.7672, 17100.814, 17206.8282, 17305.8262, 17416.0744, 17508.4092, 17617.0178, 17715.4554, 17816.758, 17920.1748, 18012.9236, 18119.7984, 18223.2248, 18324.2482, 18426.6276, 18525.0932, 18629.8976, 18733.2588, 18831.0466, 18940.1366, 19032.2696, 19131.729, 19243.4864, 19349.6932, 19442.866, 19547.9448, 19653.2798, 19754.4034, 19854.0692, 19965.1224, 20065.1774, 20158.2212, 20253.353, 20366.3264, 20463.22}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p13[] = {5908.5052, 6007.2672, 6107.347, 6208.5794, 6311.2622, 6414.5514, 6519.3376, 6625.6952, 6732.5988, 6841.3552, 6950.5972, 7061.3082, 7173.5646, 7287.109, 7401.8216, 7516.4344, 7633.3802, 7751.2962, 7870.3784, 7990.292, 8110.79, 8233.4574, 8356.6036, 8482.2712, 8607.7708, 8735.099, 8863.1858, 8993.4746, 9123.8496, 9255.6794, 9388.5448, 9522.7516, 9657.3106, 9792.6094, 9930.5642, 10068.794, 10206.7256, 10347.81, 10490.3196, 10632.0778, 10775.9916, 10920.4662, 11066.124, 11213.073, 11358.0362, 11508.1006, 11659.1716, 11808.7514, 11959.4884, 12112.1314, 12265.037, 12420.3756, 12578.933, 12734.311, 12890.0006, 13047.2144, 13207.3096, 13368.5144, 13528.024, 13689.847, 13852.7528, 14018.3168, 14180.5372, 14346.9668, 14513.5074, 14677.867, 14846.2186, 15017.4186, 15184.9716, 15356.339, 15529.2972, 15697.3578, 15871.8686, 16042.187, 16216.4094, 16389.4188, 16565.9126, 16742.3272, 16919.0042, 17094.7592, 17273.965, 17451.8342, 17634.4254, 17810.5984, 17988.9242, 18171.051, 18354.7938, 18539.466, 18721.0408, 18904.9972, 19081.867, 19271.9118, 19451.8694, 19637.9816, 19821.2922, 20013.1292, 20199.3858, 20387.8726, 20572.9514, 20770.7764, 20955.1714, 21144.751, 21329.9952, 21520.709, 21712.7016, 21906.3868, 22096.2626, 22286.0524, 22475.051, 22665.5098, 22862.8492, 23055.5294, 23249.6138, 23437.848, 23636.273, 23826.093, 24020.3296, 24213.3896, 24411.7392, 24602.9614, 24805.7952, 24998.1552, 25193.9588, 25389.0166, 25585.8392, 25780.6976, 25981.2728, 26175.977, 26376.5252, 26570.1964, 26773.387, 26962.9812, 27163.0586, 27368.164, 27565.0534, 27758.7428, 27961.1276, 28163.2324, 28362.3816, 28565.7668, 28758.644, 28956.9768, 29163.4722, 29354.7026, 29561.1186, 29767.9948, 29959.9986, 30164.0492, 30366.9818, 30562.5338, 30762.9928, 30976.1592, 31166.274, 31376.722, 31570.3734, 31770.809, 31974.8934, 32179.5286, 32387.5442, 32582.3504, 32794.076, 32989.9528, 33191.842, 33392.4684, 33595.659, 33801.8672, 34000.3414, 34200.0922, 34402.6792, 34610.0638, 34804.0084, 35011.13, 35218.669, 35418.6634, 35619.0792, 35830.6534, 36028.4966, 36229.7902, 36438.6422, 36630.7764, 36833.3102, 37048.6728, 37247.3916, 37453.5904, 37669.3614, 37854.5526, 38059.305, 38268.0936, 38470.2516, 38674.7064, 38876.167, 39068.3794, 39281.9144, 39492.8566, 39684.8628, 39898.4108, 40093.1836, 40297.6858, 40489.7086, 40717.2424}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p14[] = {11817.475, 12015.0046, 12215.3792, 12417.7504, 12623.1814, 12830.0086, 13040.0072, 13252.503, 13466.178, 13683.2738, 13902.0344, 14123.9798, 14347.394, 14573.7784, 14802.6894, 15033.6824, 15266.9134, 15502.8624, 15741.4944, 15980.7956, 16223.8916, 16468.6316, 16715.733, 16965.5726, 17217.204, 17470.666, 17727.8516, 17986.7886, 18247.6902, 18510.9632, 18775.304, 19044.7486, 19314.4408, 19587.202, 19862.2576, 20135.924, 20417.0324, 20697.9788, 20979.6112, 21265.0274, 21550.723, 21841.6906, 22132.162, 22428.1406, 22722.127, 23020.5606, 23319.7394, 23620.4014, 23925.2728, 24226.9224, 24535.581, 24845.505, 25155.9618, 25470.3828, 25785.9702, 26103.7764, 26420.4132, 26742.0186, 27062.8852, 27388.415, 27714.6024, 28042.296, 28365.4494, 28701.1526, 29031.8008, 29364.2156, 29704.497, 30037.1458, 30380.111, 30723.8168, 31059.5114, 31404.9498, 31751.6752, 32095.2686, 32444.7792, 32794.767, 33145.204, 33498.4226, 33847.6502, 34209.006, 34560.849, 34919.4838, 35274.9778, 35635.1322, 35996.3266, 36359.1394, 36722.8266, 37082.8516, 37447.7354, 37815.9606, 38191.0692, 38559.4106, 38924.8112, 39294.6726, 39663.973, 40042.261, 40416.2036, 40779.2036, 41161.6436, 41540.9014, 41921.1998, 42294.7698, 42678.5264, 43061.3464, 43432.375, 43818.432, 44198.6598, 44583.0138, 44970.4794, 45353.924, 45729.858, 46118.2224, 46511.5724, 46900.7386, 47280.6964, 47668.1472, 48055.6796, 48446.9436, 48838.7146, 49217.7296, 49613.7796, 50010.7508, 50410.0208, 50793.7886, 51190.2456, 51583.1882, 51971.0796, 52376.5338, 52763.319, 53165.5534, 53556.5594, 53948.2702, 54346.352, 54748.7914, 55138.577, 55543.4824, 55941.1748, 56333.7746, 56745.1552, 57142.7944, 57545.2236, 57935.9956, 58348.5268, 58737.5474, 59158.5962, 59542.6896, 59958.8004, 60349.3788, 60755.0212, 61147.6144, 61548.194, 61946.0696, 62348.6042, 62763.603, 63162.781, 63560.635, 63974.3482, 64366.4908, 64771.5876, 65176.7346, 65597.3916, 65995.915, 66394.0384, 66822.9396, 67203.6336, 67612.2032, 68019.0078, 68420.0388, 68821.22, 69235.8388, 69640.0724, 70055.155, 70466.357, 70863.4266, 71276.2482, 71677.0306, 72080.2006, 72493.0214, 72893.5952, 73314.5856, 73714.9852, 74125.3022, 74521.2122, 74933.6814, 75341.5904, 75743.0244, 76166.0278, 76572.1322, 76973.1028, 77381.6284, 77800.6092, 78189.328, 78607.0962, 79012.2508, 79407.8358, 79825.725, 80238.701, 80646.891, 81035.6436, 81460.0448, 81876.3884}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p15[] = {23635.0036, 24030.8034, 24431.4744, 24837.1524, 25246.7928, 25661.326, 26081.3532, 26505.2806, 26933.9892, 27367.7098, 27805.318, 28248.799, 28696.4382, 29148.8244, 29605.5138, 30066.8668, 30534.2344, 31006.32, 31480.778, 31962.2418, 32447.3324, 32938.0232, 33432.731, 33930.728, 34433.9896, 34944.1402, 35457.5588, 35974.5958, 36497.3296, 37021.9096, 37554.326, 38088.0826, 38628.8816, 39171.3192, 39723.2326, 40274.5554, 40832.3142, 41390.613, 41959.5908, 42532.5466, 43102.0344, 43683.5072, 44266.694, 44851.2822, 45440.7862, 46038.0586, 46640.3164, 47241.064, 47846.155, 48454.7396, 49076.9168, 49692.542, 50317.4778, 50939.65, 51572.5596, 52210.2906, 52843.7396, 53481.3996, 54127.236, 54770.406, 55422.6598, 56078.7958, 56736.7174, 57397.6784, 58064.5784, 58730.308, 59404.9784, 60077.0864, 60751.9158, 61444.1386, 62115.817, 62808.7742, 63501.4774, 64187.5454, 64883.6622, 65582.7468, 66274.5318, 66976.9276, 67688.7764, 68402.138, 69109.6274, 69822.9706, 70543.6108, 71265.5202, 71983.3848, 72708.4656, 73433.384, 74158.4664, 74896.4868, 75620.9564, 76362.1434, 77098.3204, 77835.7662, 78582.6114, 79323.9902, 80067.8658, 80814.9246, 81567.0136, 82310.8536, 83061.9952, 83821.4096, 84580.8608, 85335.547, 86092.5802, 86851.6506, 87612.311, 88381.2016, 89146.3296, 89907.8974, 90676.846, 91451.4152, 92224.5518, 92995.8686, 93763.5066, 94551.2796, 95315.1944, 96096.1806, 96881.0918, 97665.679, 98442.68, 99229.3002, 100011.0994, 100790.6386, 101580.1564, 102377.7484, 103152.1392, 103944.2712, 104730.216, 105528.6336, 106324.9398, 107117.6706, 107890.3988, 108695.2266, 109485.238, 110294.7876, 111075.0958, 111878.0496, 112695.2864, 113464.5486, 114270.0474, 115068.608, 115884.3626, 116673.2588, 117483.3716, 118275.097, 119085.4092, 119879.2808, 120687.5868, 121499.9944, 122284.916, 123095.9254, 123912.5038, 124709.0454, 125503.7182, 126323.259, 127138.9412, 127943.8294, 128755.646, 129556.5354, 130375.3298, 131161.4734, 131971.1962, 132787.5458, 133588.1056, 134431.351, 135220.2906, 136023.398, 136846.6558, 137667.0004, 138463.663, 139283.7154, 140074.6146, 140901.3072, 141721.8548, 142543.2322, 143356.1096, 144173.7412, 144973.0948, 145794.3162, 146609.5714, 147420.003, 148237.9784, 149050.5696, 149854.761, 150663.1966, 151494.0754, 152313.1416, 153112.6902, 153935.7206, 154746.9262, 155559.547, 156401.9746, 157228.7036, 158008.7254, 158820.75, 159646.9184, 160470.4458, 161279.5348, 162093.3114, 162918.542, 163729.2842}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p16[] = {47271.0, 48062.3584, 48862.7074, 49673.152, 50492.8416, 51322.9514, 52161.03, 53009.407, 53867.6348, 54734.206, 55610.5144, 56496.2096, 57390.795, 58297.268, 59210.6448, 60134.665, 61068.0248, 62010.4472, 62962.5204, 63923.5742, 64895.0194, 65876.4182, 66862.6136, 67862.6968, 68868.8908, 69882.8544, 70911.271, 71944.0924, 72990.0326, 74040.692, 75100.6336, 76174.7826, 77252.5998, 78340.2974, 79438.2572, 80545.4976, 81657.2796, 82784.6336, 83915.515, 85059.7362, 86205.9368, 87364.4424, 88530.3358, 89707.3744, 90885.9638, 92080.197, 93275.5738, 94479.391, 95695.918, 96919.2236, 98148.4602, 99382.3474, 100625.6974, 101878.0284, 103141.6278, 104409.4588, 105686.2882, 106967.5402, 108261.6032, 109548.1578, 110852.0728, 112162.231, 113479.0072, 114806.2626, 116137.9072, 117469.5048, 118813.5186, 120165.4876, 121516.2556, 122875.766, 124250.5444, 125621.2222, 127003.2352, 128387.848, 129775.2644, 131181.7776, 132577.3086, 133979.9458, 135394.1132, 136800.9078, 138233.217, 139668.5308, 141085.212, 142535.2122, 143969.0684, 145420.2872, 146878.1542, 148332.7572, 149800.3202, 151269.66, 152743.6104, 154213.0948, 155690.288, 157169.4246, 158672.1756, 160160.059, 161650.6854, 163145.7772, 164645.6726, 166159.1952, 167682.1578, 169177.3328, 170700.0118, 172228.8964, 173732.6664, 175265.5556, 176787.799, 178317.111, 179856.6914, 181400.865, 182943.4612, 184486.742, 186033.4698, 187583.7886, 189148.1868, 190688.4526, 192250.1926, 193810.9042, 195354.2972, 196938.7682, 198493.5898, 200079.2824, 201618.912, 203205.5492, 204765.5798, 206356.1124, 207929.3064, 209498.7196, 211086.229, 212675.1324, 214256.7892, 215826.2392, 217412.8474, 218995.6724, 220618.6038, 222207.1166, 223781.0364, 225387.4332, 227005.7928, 228590.4336, 230217.8738, 231805.1054, 233408.9, 234995.3432, 236601.4956, 238190.7904, 239817.2548, 241411.2832, 243002.4066, 244640.1884, 246255.3128, 247849.3508, 249479.9734, 251106.8822, 252705.027, 254332.9242, 255935.129, 257526.9014, 259154.772, 260777.625, 262390.253, 264004.4906, 265643.59, 267255.4076, 268873.426, 270470.7252, 272106.4804, 273722.4456, 275337.794, 276945.7038, 278592.9154, 280204.3726, 281841.1606, 283489.171, 285130.1716, 286735.3362, 288364.7164, 289961.1814, 291595.5524, 293285.683, 294899.6668, 296499.3434, 298128.0462, 299761.8946, 301394.2424, 302997.6748, 304615.1478, 306269.7724, 307886.114, 309543.1028, 311153.2862, 312782.8546, 314421.2008, 316033.2438, 317692.9636, 319305.2648, 320948.7406, 322566.3364, 324228.4224, 325847.1542}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p17[] = {94542.0, 96125.811, 97728.019, 99348.558, 100987.9705, 102646.7565, 104324.5125, 106021.7435, 107736.7865, 109469.272, 111223.9465, 112995.219, 114787.432, 116593.152, 118422.71, 120267.2345, 122134.6765, 124020.937, 125927.2705, 127851.255, 129788.9485, 131751.016, 133726.8225, 135722.592, 137736.789, 139770.568, 141821.518, 143891.343, 145982.1415, 148095.387, 150207.526, 152355.649, 154515.6415, 156696.05, 158887.7575, 161098.159, 163329.852, 165569.053, 167837.4005, 170121.6165, 172420.4595, 174732.6265, 177062.77, 179412.502, 181774.035, 184151.939, 186551.6895, 188965.691, 191402.8095, 193857.949, 196305.0775, 198774.6715, 201271.2585, 203764.78, 206299.3695, 208818.1365, 211373.115, 213946.7465, 216532.076, 219105.541, 221714.5375, 224337.5135, 226977.5125, 229613.0655, 232270.2685, 234952.2065, 237645.3555, 240331.1925, 243034.517, 245756.0725, 248517.6865, 251232.737, 254011.3955, 256785.995, 259556.44, 262368.335, 265156.911, 267965.266, 270785.583, 273616.0495, 276487.4835, 279346.639, 282202.509, 285074.3885, 287942.2855, 290856.018, 293774.0345, 296678.5145, 299603.6355, 302552.6575, 305492.9785, 308466.8605, 311392.581, 314347.538, 317319.4295, 320285.9785, 323301.7325, 326298.3235, 329301.3105, 332301.987, 335309.791, 338370.762, 341382.923, 344431.1265, 347464.1545, 350507.28, 353619.2345, 356631.2005, 359685.203, 362776.7845, 365886.488, 368958.2255, 372060.6825, 375165.4335, 378237.935, 381328.311, 384430.5225, 387576.425, 390683.242, 393839.648, 396977.8425, 400101.9805, 403271.296, 406409.8425, 409529.5485, 412678.7, 415847.423, 419020.8035, 422157.081, 425337.749, 428479.6165, 431700.902, 434893.1915, 438049.582, 441210.5415, 444379.2545, 447577.356, 450741.931, 453959.548, 457137.0935, 460329.846, 463537.4815, 466732.3345, 469960.5615, 473164.681, 476347.6345, 479496.173, 482813.1645, 486025.6995, 489249.4885, 492460.1945, 495675.8805, 498908.0075, 502131.802, 505374.3855, 508550.9915, 511806.7305, 515026.776, 518217.0005, 521523.9855, 524705.9855, 527950.997, 531210.0265, 534472.497, 537750.7315, 540926.922, 544207.094, 547429.4345, 550666.3745, 553975.3475, 557150.7185, 560399.6165, 563662.697, 566916.7395, 570146.1215, 573447.425, 576689.6245, 579874.5745, 583202.337, 586503.0255, 589715.635, 592910.161, 596214.3885, 599488.035, 602740.92, 605983.0685, 609248.67, 612491.3605, 615787.912, 619107.5245, 622307.9555, 625577.333, 628840.4385, 632085.2155, 635317.6135, 638691.7195, 641887.467, 645139.9405, 648441.546, 651666.252, 654941.845}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __raw_estimate_data_p18[] = {189084.0, 192250.913, 195456.774, 198696.946, 201977.762, 205294.444, 208651.754, 212042.099, 215472.269, 218941.91, 222443.912, 225996.845, 229568.199, 233193.568, 236844.457, 240543.233, 244279.475, 248044.27, 251854.588, 255693.2, 259583.619, 263494.621, 267445.385, 271454.061, 275468.769, 279549.456, 283646.446, 287788.198, 291966.099, 296181.164, 300431.469, 304718.618, 309024.004, 313393.508, 317760.803, 322209.731, 326675.061, 331160.627, 335654.47, 340241.442, 344841.833, 349467.132, 354130.629, 358819.432, 363574.626, 368296.587, 373118.482, 377914.93, 382782.301, 387680.669, 392601.981, 397544.323, 402529.115, 407546.018, 412593.658, 417638.657, 422762.865, 427886.169, 433017.167, 438213.273, 443441.254, 448692.421, 453937.533, 459239.049, 464529.569, 469910.083, 475274.03, 480684.473, 486070.26, 491515.237, 496995.651, 502476.617, 507973.609, 513497.19, 519083.233, 524726.509, 530305.505, 535945.728, 541584.404, 547274.055, 552967.236, 558667.862, 564360.216, 570128.148, 575965.08, 581701.952, 587532.523, 593361.144, 599246.128, 605033.418, 610958.779, 616837.117, 622772.818, 628672.04, 634675.369, 640574.831, 646585.739, 652574.547, 658611.217, 664642.684, 670713.914, 676737.681, 682797.313, 688837.897, 694917.874, 701009.882, 707173.648, 713257.254, 719415.392, 725636.761, 731710.697, 737906.209, 744103.074, 750313.39, 756504.185, 762712.579, 768876.985, 775167.859, 781359.0, 787615.959, 793863.597, 800245.477, 806464.582, 812785.294, 819005.925, 825403.057, 831676.197, 837936.284, 844266.968, 850642.711, 856959.756, 863322.774, 869699.931, 876102.478, 882355.787, 888694.463, 895159.952, 901536.143, 907872.631, 914293.672, 920615.14, 927130.974, 933409.404, 939922.178, 946331.47, 952745.93, 959209.264, 965590.224, 972077.284, 978501.961, 984953.19, 991413.271, 997817.479, 1004222.658, 1010725.676, 1017177.138, 1023612.529, 1030098.236, 1036493.719, 1043112.207, 1049537.036, 1056008.096, 1062476.184, 1068942.337, 1075524.95, 1081932.864, 1088426.025, 1094776.005, 1101327.448, 1107901.673, 1114423.639, 1120884.602, 1127324.923, 1133794.24, 1140328.886, 1146849.376, 1153346.682, 1159836.502, 1166478.703, 1172953.304, 1179391.502, 1185950.982, 1192544.052, 1198913.41, 1205430.994, 1212015.525, 1218674.042, 1225121.683, 1231551.101, 1238126.379, 1244673.795, 1251260.649, 1257697.86, 1264320.983, 1270736.319, 1277274.694, 1283804.95, 1290211.514, 1296858.568, 1303455.691}; - -//! @brief Get raw estimate data array for a given precision -//! -//! @param __precision The precision value (4-18) -//! @return Pointer to the raw estimate data array for the given precision -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const double* __raw_estimate_data(::cuda::std::int32_t __precision) noexcept { - switch (__precision) { - case 4: return __raw_estimate_data_p4; - case 5: return __raw_estimate_data_p5; - case 6: return __raw_estimate_data_p6; - case 7: return __raw_estimate_data_p7; - case 8: return __raw_estimate_data_p8; - case 9: return __raw_estimate_data_p9; - case 10: return __raw_estimate_data_p10; - case 11: return __raw_estimate_data_p11; - case 12: return __raw_estimate_data_p12; - case 13: return __raw_estimate_data_p13; - case 14: return __raw_estimate_data_p14; - case 15: return __raw_estimate_data_p15; - case 16: return __raw_estimate_data_p16; - case 17: return __raw_estimate_data_p17; - case 18: return __raw_estimate_data_p18; - default: return nullptr; - } -} - -//! @brief Get size of raw estimate data array for a given precision -//! -//! @param __precision The precision value (4-18) -//! @return Size of the raw estimate data array for the given precision -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::size_t __raw_estimate_data_size(::cuda::std::int32_t __precision) noexcept { - constexpr auto __size_of_double = sizeof(double); - switch (__precision) { - case 4: return sizeof(__raw_estimate_data_p4) / __size_of_double; - case 5: return sizeof(__raw_estimate_data_p5) / __size_of_double; - case 6: return sizeof(__raw_estimate_data_p6) / __size_of_double; - case 7: return sizeof(__raw_estimate_data_p7) / __size_of_double; - case 8: return sizeof(__raw_estimate_data_p8) / __size_of_double; - case 9: return sizeof(__raw_estimate_data_p9) / __size_of_double; - case 10: return sizeof(__raw_estimate_data_p10) / __size_of_double; - case 11: return sizeof(__raw_estimate_data_p11) / __size_of_double; - case 12: return sizeof(__raw_estimate_data_p12) / __size_of_double; - case 13: return sizeof(__raw_estimate_data_p13) / __size_of_double; - case 14: return sizeof(__raw_estimate_data_p14) / __size_of_double; - case 15: return sizeof(__raw_estimate_data_p15) / __size_of_double; - case 16: return sizeof(__raw_estimate_data_p16) / __size_of_double; - case 17: return sizeof(__raw_estimate_data_p17) / __size_of_double; - case 18: return sizeof(__raw_estimate_data_p18) / __size_of_double; - default: return 0; - } -} - -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p4[] = {10.0, 9.717, 9.207, 8.7896, 8.2882, 7.8204, 7.3772, 6.9342, 6.5202, 6.161, 5.7722, 5.4636, 5.0396, 4.6766, 4.3566, 4.0454, 3.7936, 3.4856, 3.2666, 2.9946, 2.766, 2.4692, 2.3638, 2.0764, 1.7864, 1.7602, 1.4814, 1.433, 1.2926, 1.0664, 0.999600000000001, 0.7956, 0.5366, 0.589399999999998, 0.573799999999999, 0.269799999999996, 0.368200000000002, 0.0544000000000011, 0.234200000000001, 0.0108000000000033, -0.203400000000002, -0.0701999999999998, -0.129600000000003, -0.364199999999997, -0.480600000000003, -0.226999999999997, -0.322800000000001, -0.382599999999996, -0.511200000000002, -0.669600000000003, -0.749400000000001, -0.500399999999999, -0.617600000000003, -0.6922, -0.601599999999998, -0.416200000000003, -0.338200000000001, -0.782600000000002, -0.648600000000002, -0.919800000000002, -0.851799999999997, -0.962400000000002, -0.6402, -1.1922, -1.0256, -1.086, -1.21899999999999, -0.819400000000002, -0.940600000000003, -1.1554, -1.2072, -1.1752, -1.16759999999999, -1.14019999999999, -1.3754, -1.29859999999999, -1.607, -1.3292, -1.7606}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p5[] = {22.0, 21.1194, 20.8208, 20.2318, 19.77, 19.2436, 18.7774, 18.2848, 17.8224, 17.3742, 16.9336, 16.503, 16.0494, 15.6292, 15.2124, 14.798, 14.367, 13.9728, 13.5944, 13.217, 12.8438, 12.3696, 12.0956, 11.7044, 11.324, 11.0668, 10.6698, 10.3644, 10.049, 9.6918, 9.4146, 9.082, 8.687, 8.5398, 8.2462, 7.857, 7.6606, 7.4168, 7.1248, 6.9222, 6.6804, 6.447, 6.3454, 5.9594, 5.7636, 5.5776, 5.331, 5.19, 4.9676, 4.7564, 4.5314, 4.4442, 4.3708, 3.9774, 3.9624, 3.8796, 3.755, 3.472, 3.2076, 3.1024, 2.8908, 2.7338, 2.7728, 2.629, 2.413, 2.3266, 2.1524, 2.2642, 2.1806, 2.0566, 1.9192, 1.7598, 1.3516, 1.5802, 1.43859999999999, 1.49160000000001, 1.1524, 1.1892, 0.841399999999993, 0.879800000000003, 0.837599999999995, 0.469800000000006, 0.765600000000006, 0.331000000000003, 0.591399999999993, 0.601200000000006, 0.701599999999999, 0.558199999999999, 0.339399999999998, 0.354399999999998, 0.491200000000006, 0.308000000000007, 0.355199999999996, -0.0254000000000048, 0.205200000000005, -0.272999999999996, 0.132199999999997, 0.394400000000005, -0.241200000000006, 0.242000000000004, 0.191400000000002, 0.253799999999998, -0.122399999999999, -0.370800000000003, 0.193200000000004, -0.0848000000000013, 0.0867999999999967, -0.327200000000005, -0.285600000000002, 0.311400000000006, -0.128399999999999, -0.754999999999995, -0.209199999999996, -0.293599999999998, -0.364000000000004, -0.253600000000006, -0.821200000000005, -0.253600000000006, -0.510400000000004, -0.383399999999995, -0.491799999999998, -0.220200000000006, -0.0972000000000008, -0.557400000000001, -0.114599999999996, -0.295000000000002, -0.534800000000004, 0.346399999999988, -0.65379999999999, 0.0398000000000138, 0.0341999999999985, -0.995800000000003, -0.523400000000009, -0.489000000000004, -0.274799999999999, -0.574999999999989, -0.482799999999997, 0.0571999999999946, -0.330600000000004, -0.628800000000012, -0.140199999999993, -0.540600000000012, -0.445999999999998, -0.599400000000003, -0.262599999999992, 0.163399999999996, -0.100599999999986, -0.39500000000001, -1.06960000000001, -0.836399999999998, -0.753199999999993, -0.412399999999991, -0.790400000000005, -0.29679999999999, -0.28540000000001, -0.193000000000012, -0.0772000000000048, -0.962799999999987, -0.414800000000014}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p6[] = {45.0, 44.1902, 43.271, 42.8358, 41.8142, 41.2854, 40.317, 39.354, 38.8924, 37.9436, 37.4596, 36.5262, 35.6248, 35.1574, 34.2822, 33.837, 32.9636, 32.074, 31.7042, 30.7976, 30.4772, 29.6564, 28.7942, 28.5004, 27.686, 27.291, 26.5672, 25.8556, 25.4982, 24.8204, 24.4252, 23.7744, 23.0786, 22.8344, 22.0294, 21.8098, 21.0794, 20.5732, 20.1878, 19.5648, 19.2902, 18.6784, 18.3352, 17.8946, 17.3712, 17.0852, 16.499, 16.2686, 15.6844, 15.2234, 14.9732, 14.3356, 14.2286, 13.7262, 13.3284, 13.1048, 12.5962, 12.3562, 12.1272, 11.4184, 11.4974, 11.0822, 10.856, 10.48, 10.2834, 10.0208, 9.637, 9.51739999999999, 9.05759999999999, 8.74760000000001, 8.42700000000001, 8.1326, 8.2372, 8.2788, 7.6776, 7.79259999999999, 7.1952, 6.9564, 6.6454, 6.87, 6.5428, 6.19999999999999, 6.02940000000001, 5.62780000000001, 5.6782, 5.792, 5.35159999999999, 5.28319999999999, 5.0394, 5.07480000000001, 4.49119999999999, 4.84899999999999, 4.696, 4.54040000000001, 4.07300000000001, 4.37139999999999, 3.7216, 3.7328, 3.42080000000001, 3.41839999999999, 3.94239999999999, 3.27719999999999, 3.411, 3.13079999999999, 2.76900000000001, 2.92580000000001, 2.68279999999999, 2.75020000000001, 2.70599999999999, 2.3886, 3.01859999999999, 2.45179999999999, 2.92699999999999, 2.41720000000001, 2.41139999999999, 2.03299999999999, 2.51240000000001, 2.5564, 2.60079999999999, 2.41720000000001, 1.80439999999999, 1.99700000000001, 2.45480000000001, 1.8948, 2.2346, 2.30860000000001, 2.15479999999999, 1.88419999999999, 1.6508, 0.677199999999999, 1.72540000000001, 1.4752, 1.72280000000001, 1.66139999999999, 1.16759999999999, 1.79300000000001, 1.00059999999999, 0.905200000000008, 0.659999999999997, 1.55879999999999, 1.1636, 0.688199999999995, 0.712600000000009, 0.450199999999995, 1.1978, 0.975599999999986, 0.165400000000005, 1.727, 1.19739999999999, -0.252600000000001, 1.13460000000001, 1.3048, 1.19479999999999, 0.313400000000001, 0.878999999999991, 1.12039999999999, 0.853000000000009, 1.67920000000001, 0.856999999999999, 0.448599999999999, 1.2362, 0.953399999999988, 1.02859999999998, 0.563199999999995, 0.663000000000011, 0.723000000000013, 0.756599999999992, 0.256599999999992, -0.837600000000009, 0.620000000000005, 0.821599999999989, 0.216600000000028, 0.205600000000004, 0.220199999999977, 0.372599999999977, 0.334400000000016, 0.928400000000011, 0.972800000000007, 0.192400000000021, 0.487199999999973, -0.413000000000011, 0.807000000000016, 0.120600000000024, 0.769000000000005, 0.870799999999974, 0.66500000000002, 0.118200000000002, 0.401200000000017, 0.635199999999998, 0.135400000000004, 0.175599999999974, 1.16059999999999, 0.34620000000001, 0.521400000000028, -0.586599999999976, -1.16480000000001, 0.968399999999974, 0.836999999999989, 0.779600000000016, 0.985799999999983}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p7[] = {91.0, 89.4934, 87.9758, 86.4574, 84.9718, 83.4954, 81.5302, 80.0756, 78.6374, 77.1782, 75.7888, 73.9522, 72.592, 71.2532, 69.9086, 68.5938, 66.9474, 65.6796, 64.4394, 63.2176, 61.9768, 60.4214, 59.2528, 58.0102, 56.8658, 55.7278, 54.3044, 53.1316, 52.093, 51.0032, 49.9092, 48.6306, 47.5294, 46.5756, 45.6508, 44.662, 43.552, 42.3724, 41.617, 40.5754, 39.7872, 38.8444, 37.7988, 36.8606, 36.2118, 35.3566, 34.4476, 33.5882, 32.6816, 32.0824, 31.0258, 30.6048, 29.4436, 28.7274, 27.957, 27.147, 26.4364, 25.7592, 25.3386, 24.781, 23.8028, 23.656, 22.6544, 21.996, 21.4718, 21.1544, 20.6098, 19.5956, 19.0616, 18.5758, 18.4878, 17.5244, 17.2146, 16.724, 15.8722, 15.5198, 15.0414, 14.941, 14.9048, 13.87, 13.4304, 13.028, 12.4708, 12.37, 12.0624, 11.4668, 11.5532, 11.4352, 11.2564, 10.2744, 10.2118, 9.74720000000002, 10.1456, 9.2928, 8.75040000000001, 8.55279999999999, 8.97899999999998, 8.21019999999999, 8.18340000000001, 7.3494, 7.32499999999999, 7.66140000000001, 6.90300000000002, 7.25439999999998, 6.9042, 7.21499999999997, 6.28640000000001, 6.08139999999997, 6.6764, 6.30099999999999, 5.13900000000001, 5.65800000000002, 5.17320000000001, 4.59019999999998, 4.9538, 5.08280000000002, 4.92200000000003, 4.99020000000002, 4.7328, 5.4538, 4.11360000000002, 4.22340000000003, 4.08780000000002, 3.70800000000003, 4.15559999999999, 4.18520000000001, 3.63720000000001, 3.68220000000002, 3.77960000000002, 3.6078, 2.49160000000001, 3.13099999999997, 2.5376, 3.19880000000001, 3.21100000000001, 2.4502, 3.52820000000003, 2.91199999999998, 3.04480000000001, 2.7432, 2.85239999999999, 2.79880000000003, 2.78579999999999, 1.88679999999999, 2.98860000000002, 2.50639999999999, 1.91239999999999, 2.66160000000002, 2.46820000000002, 1.58199999999999, 1.30399999999997, 2.27379999999999, 2.68939999999998, 1.32900000000001, 3.10599999999999, 1.69080000000002, 2.13740000000001, 2.53219999999999, 1.88479999999998, 1.33240000000001, 1.45119999999997, 1.17899999999997, 2.44119999999998, 1.60659999999996, 2.16700000000003, 0.77940000000001, 2.37900000000002, 2.06700000000001, 1.46000000000004, 2.91160000000002, 1.69200000000001, 0.954600000000028, 2.49300000000005, 2.2722, 1.33500000000004, 2.44899999999996, 1.20140000000004, 3.07380000000001, 2.09739999999999, 2.85640000000001, 2.29960000000005, 2.40899999999999, 1.97040000000004, 0.809799999999996, 1.65279999999996, 2.59979999999996, 0.95799999999997, 2.06799999999998, 2.32780000000002, 4.20159999999998, 1.96320000000003, 1.86400000000003, 1.42999999999995, 3.77940000000001, 1.27200000000005, 1.86440000000005, 2.20600000000002, 3.21900000000005, 1.5154, 2.61019999999996}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p8[] = {183.2152, 180.2454, 177.2096, 173.6652, 170.6312, 167.6822, 164.249, 161.3296, 158.0038, 155.2074, 152.4612, 149.27, 146.5178, 143.4412, 140.8032, 138.1634, 135.1688, 132.6074, 129.6946, 127.2664, 124.8228, 122.0432, 119.6824, 116.9464, 114.6268, 112.2626, 109.8376, 107.4034, 104.8956, 102.8522, 100.7638, 98.3552, 96.3556, 93.7526, 91.9292, 89.8954, 87.8198, 85.7668, 83.298, 81.6688, 79.9466, 77.9746, 76.1672, 74.3474, 72.3028, 70.8912, 69.114, 67.4646, 65.9744, 64.4092, 62.6022, 60.843, 59.5684, 58.1652, 56.5426, 55.4152, 53.5388, 52.3592, 51.1366, 49.486, 48.3918, 46.5076, 45.509, 44.3834, 43.3498, 42.0668, 40.7346, 40.1228, 38.4528, 37.7, 36.644, 36.0518, 34.5774, 33.9068, 32.432, 32.1666, 30.434, 29.6644, 28.4894, 27.6312, 26.3804, 26.292, 25.5496000000001, 25.0234, 24.8206, 22.6146, 22.4188, 22.117, 20.6762, 20.6576, 19.7864, 19.509, 18.5334, 17.9204, 17.772, 16.2924, 16.8654, 15.1836, 15.745, 15.1316, 15.0386, 14.0136, 13.6342, 12.6196, 12.1866, 12.4281999999999, 11.3324, 10.4794000000001, 11.5038, 10.129, 9.52800000000002, 10.3203999999999, 9.46299999999997, 9.79280000000006, 9.12300000000005, 8.74180000000001, 9.2192, 7.51020000000005, 7.60659999999996, 7.01840000000004, 7.22239999999999, 7.40139999999997, 6.76179999999999, 7.14359999999999, 5.65060000000005, 5.63779999999997, 5.76599999999996, 6.75139999999999, 5.57759999999996, 3.73220000000003, 5.8048, 5.63019999999995, 4.93359999999996, 3.47979999999995, 4.33879999999999, 3.98940000000005, 3.81960000000004, 3.31359999999995, 3.23080000000004, 3.4588, 3.08159999999998, 3.4076, 3.00639999999999, 2.38779999999997, 2.61900000000003, 1.99800000000005, 3.34820000000002, 2.95060000000001, 0.990999999999985, 2.11440000000005, 2.20299999999997, 2.82219999999995, 2.73239999999998, 2.7826, 3.76660000000004, 2.26480000000004, 2.31280000000004, 2.40819999999997, 2.75360000000001, 3.33759999999995, 2.71559999999999, 1.7478000000001, 1.42920000000004, 2.39300000000003, 2.22779999999989, 2.34339999999997, 0.87259999999992, 3.88400000000001, 1.80600000000004, 1.91759999999999, 1.16779999999994, 1.50320000000011, 2.52500000000009, 0.226400000000012, 2.31500000000005, 0.930000000000064, 1.25199999999995, 2.14959999999996, 0.0407999999999902, 2.5447999999999, 1.32960000000003, 0.197400000000016, 2.52620000000002, 3.33279999999991, -1.34300000000007, 0.422199999999975, 0.917200000000093, 1.12920000000008, 1.46060000000011, 1.45779999999991, 2.8728000000001, 3.33359999999993, -1.34079999999994, 1.57680000000005, 0.363000000000056, 1.40740000000005, 0.656600000000026, 0.801400000000058, -0.454600000000028, 1.51919999999996}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p9[] = {368.0, 361.8294, 355.2452, 348.6698, 342.1464, 336.2024, 329.8782, 323.6598, 317.462, 311.2826, 305.7102, 299.7416, 293.9366, 288.1046, 282.285, 277.0668, 271.306, 265.8448, 260.301, 254.9886, 250.2422, 244.8138, 239.7074, 234.7428, 229.8402, 225.1664, 220.3534, 215.594, 210.6886, 205.7876, 201.65, 197.228, 192.8036, 188.1666, 184.0818, 180.0824, 176.2574, 172.302, 168.1644, 164.0056, 160.3802, 156.7192, 152.5234, 149.2084, 145.831, 142.485, 139.1112, 135.4764, 131.76, 129.3368, 126.5538, 122.5058, 119.2646, 116.5902, 113.3818, 110.8998, 107.9532, 105.2062, 102.2798, 99.4728, 96.9582, 94.3292, 92.171, 89.7809999999999, 87.5716, 84.7048, 82.5322, 79.875, 78.3972, 75.3464, 73.7274, 71.2834, 70.1444, 68.4263999999999, 66.0166, 64.018, 62.0437999999999, 60.3399999999999, 58.6856, 57.9836, 55.0311999999999, 54.6769999999999, 52.3188, 51.4846, 49.4423999999999, 47.739, 46.1487999999999, 44.9202, 43.4059999999999, 42.5342000000001, 41.2834, 38.8954000000001, 38.3286000000001, 36.2146, 36.6684, 35.9946, 33.123, 33.4338, 31.7378000000001, 29.076, 28.9692, 27.4964, 27.0998, 25.9864, 26.7754, 24.3208, 23.4838, 22.7388000000001, 24.0758000000001, 21.9097999999999, 20.9728, 19.9228000000001, 19.9292, 16.617, 17.05, 18.2996000000001, 15.6128000000001, 15.7392, 14.5174, 13.6322, 12.2583999999999, 13.3766000000001, 11.423, 13.1232, 9.51639999999998, 10.5938000000001, 9.59719999999993, 8.12220000000002, 9.76739999999995, 7.50440000000003, 7.56999999999994, 6.70440000000008, 6.41419999999994, 6.71019999999999, 5.60940000000005, 4.65219999999999, 6.84099999999989, 3.4072000000001, 3.97859999999991, 3.32760000000007, 5.52160000000003, 3.31860000000006, 2.06940000000009, 4.35400000000004, 1.57500000000005, 0.280799999999999, 2.12879999999996, -0.214799999999968, -0.0378000000000611, -0.658200000000079, 0.654800000000023, -0.0697999999999865, 0.858400000000074, -2.52700000000004, -2.1751999999999, -3.35539999999992, -1.04019999999991, -0.651000000000067, -2.14439999999991, -1.96659999999997, -3.97939999999994, -0.604400000000169, -3.08260000000018, -3.39159999999993, -5.29640000000018, -5.38920000000007, -5.08759999999984, -4.69900000000007, -5.23720000000003, -3.15779999999995, -4.97879999999986, -4.89899999999989, -7.48880000000008, -5.94799999999987, -5.68060000000014, -6.67180000000008, -4.70499999999993, -7.27779999999984, -4.6579999999999, -4.4362000000001, -4.32139999999981, -5.18859999999995, -6.66879999999992, -6.48399999999992, -5.1260000000002, -4.4032000000002, -6.13500000000022, -5.80819999999994, -4.16719999999987, -4.15039999999999, -7.45600000000013, -7.24080000000004, -9.83179999999993, -5.80420000000004, -8.6561999999999, -6.99940000000015, -10.5473999999999, -7.34139999999979, -6.80999999999995, -6.29719999999998, -6.23199999999997}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p10[] = {737.1256, 724.4234, 711.1064, 698.4732, 685.4636, 673.0644, 660.488, 647.9654, 636.0832, 623.7864, 612.1992, 600.2176, 588.5228, 577.1716, 565.7752, 554.899, 543.6126, 532.6492, 521.9474, 511.5214, 501.1064, 490.6364, 480.2468, 470.4588, 460.3832, 451.0584, 440.8606, 431.3868, 422.5062, 413.1862, 404.463, 395.339, 386.1936, 378.1292, 369.1854, 361.2908, 353.3324, 344.8518, 337.5204, 329.4854, 321.9318, 314.552, 306.4658, 299.4256, 292.849, 286.152, 278.8956, 271.8792, 265.118, 258.62, 252.5132, 245.9322, 239.7726, 233.6086, 227.5332, 222.5918, 216.4294, 210.7662, 205.4106, 199.7338, 194.9012, 188.4486, 183.1556, 178.6338, 173.7312, 169.6264, 163.9526, 159.8742, 155.8326, 151.1966, 147.5594, 143.07, 140.037, 134.1804, 131.071, 127.4884, 124.0848, 120.2944, 117.333, 112.9626, 110.2902, 107.0814, 103.0334, 99.4832000000001, 96.3899999999999, 93.7202000000002, 90.1714000000002, 87.2357999999999, 85.9346, 82.8910000000001, 80.0264000000002, 78.3834000000002, 75.1543999999999, 73.8683999999998, 70.9895999999999, 69.4367999999999, 64.8701999999998, 65.0408000000002, 61.6738, 59.5207999999998, 57.0158000000001, 54.2302, 53.0962, 50.4985999999999, 52.2588000000001, 47.3914, 45.6244000000002, 42.8377999999998, 43.0072, 40.6516000000001, 40.2453999999998, 35.2136, 36.4546, 33.7849999999999, 33.2294000000002, 32.4679999999998, 30.8670000000002, 28.6507999999999, 28.9099999999999, 27.5983999999999, 26.1619999999998, 24.5563999999999, 23.2328000000002, 21.9484000000002, 21.5902000000001, 21.3346000000001, 17.7031999999999, 20.6111999999998, 19.5545999999999, 15.7375999999999, 17.0720000000001, 16.9517999999998, 15.326, 13.1817999999998, 14.6925999999999, 13.0859999999998, 13.2754, 10.8697999999999, 11.248, 7.3768, 4.72339999999986, 7.97899999999981, 8.7503999999999, 7.68119999999999, 9.7199999999998, 7.73919999999998, 5.6224000000002, 7.44560000000001, 6.6601999999998, 5.9058, 4.00199999999995, 4.51699999999983, 4.68240000000014, 3.86220000000003, 5.13639999999987, 5.98500000000013, 2.47719999999981, 2.61999999999989, 1.62800000000016, 4.65000000000009, 0.225599999999758, 0.831000000000131, -0.359400000000278, 1.27599999999984, -2.92559999999958, -0.0303999999996449, 2.37079999999969, -2.0033999999996, 0.804600000000391, 0.30199999999968, 1.1247999999996, -2.6880000000001, 0.0321999999996478, -1.18099999999959, -3.9402, -1.47940000000017, -0.188400000000001, -2.10720000000038, -2.04159999999956, -3.12880000000041, -4.16160000000036, -0.612799999999879, -3.48719999999958, -8.17900000000009, -5.37780000000021, -4.01379999999972, -5.58259999999973, -5.73719999999958, -7.66799999999967, -5.69520000000011, -1.1247999999996, -5.58520000000044, -8.04560000000038, -4.64840000000004, -11.6468000000004, -7.97519999999986, -5.78300000000036, -7.67420000000038, -10.6328000000003, -9.81720000000041}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p11[] = {1476.0, 1449.6014, 1423.5802, 1397.7942, 1372.3042, 1347.2062, 1321.8402, 1297.2292, 1272.9462, 1248.9926, 1225.3026, 1201.4252, 1178.0578, 1155.6092, 1132.626, 1110.5568, 1088.527, 1066.5154, 1045.1874, 1024.3878, 1003.37, 982.1972, 962.5728, 942.1012, 922.9668, 903.292, 884.0772, 864.8578, 846.6562, 828.041, 809.714, 792.3112, 775.1806, 757.9854, 740.656, 724.346, 707.5154, 691.8378, 675.7448, 659.6722, 645.5722, 630.1462, 614.4124, 600.8728, 585.898, 572.408, 558.4926, 544.4938, 531.6776, 517.282, 505.7704, 493.1012, 480.7388, 467.6876, 456.1872, 445.5048, 433.0214, 420.806, 411.409, 400.4144, 389.4294, 379.2286, 369.651, 360.6156, 350.337, 342.083, 332.1538, 322.5094, 315.01, 305.6686, 298.1678, 287.8116, 280.9978, 271.9204, 265.3286, 257.5706, 249.6014, 242.544, 235.5976, 229.583, 220.9438, 214.672, 208.2786, 201.8628, 195.1834, 191.505, 186.1816, 178.5188, 172.2294, 167.8908, 161.0194, 158.052, 151.4588, 148.1596, 143.4344, 138.5238, 133.13, 127.6374, 124.8162, 118.7894, 117.3984, 114.6078, 109.0858, 105.1036, 103.6258, 98.6018000000004, 95.7618000000002, 93.5821999999998, 88.5900000000001, 86.9992000000002, 82.8800000000001, 80.4539999999997, 74.6981999999998, 74.3644000000004, 73.2914000000001, 65.5709999999999, 66.9232000000002, 65.1913999999997, 62.5882000000001, 61.5702000000001, 55.7035999999998, 56.1764000000003, 52.7596000000003, 53.0302000000001, 49.0609999999997, 48.4694, 44.933, 46.0474000000004, 44.7165999999997, 41.9416000000001, 39.9207999999999, 35.6328000000003, 35.5276000000003, 33.1934000000001, 33.2371999999996, 33.3864000000003, 33.9228000000003, 30.2371999999996, 29.1373999999996, 25.2272000000003, 24.2942000000003, 19.8338000000003, 18.9005999999999, 23.0907999999999, 21.8544000000002, 19.5176000000001, 15.4147999999996, 16.9314000000004, 18.6737999999996, 12.9877999999999, 14.3688000000002, 12.0447999999997, 15.5219999999999, 12.5299999999997, 14.5940000000001, 14.3131999999996, 9.45499999999993, 12.9441999999999, 3.91139999999996, 13.1373999999996, 5.44720000000052, 9.82779999999912, 7.87279999999919, 3.67760000000089, 5.46980000000076, 5.55099999999948, 5.65979999999945, 3.89439999999922, 3.1275999999998, 5.65140000000065, 6.3062000000009, 3.90799999999945, 1.87060000000019, 5.17020000000048, 2.46680000000015, 0.770000000000437, -3.72340000000077, 1.16400000000067, 8.05340000000069, 0.135399999999208, 2.15940000000046, 0.766999999999825, 1.0594000000001, 3.15500000000065, -0.287399999999252, 2.37219999999979, -2.86620000000039, -1.63199999999961, -2.22979999999916, -0.15519999999924, -1.46039999999994, -0.262199999999211, -2.34460000000036, -2.8078000000005, -3.22179999999935, -5.60159999999996, -8.42200000000048, -9.43740000000071, 0.161799999999857, -10.4755999999998, -10.0823999999993}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p12[] = {2953.0, 2900.4782, 2848.3568, 2796.3666, 2745.324, 2694.9598, 2644.648, 2595.539, 2546.1474, 2498.2576, 2450.8376, 2403.6076, 2357.451, 2311.38, 2266.4104, 2221.5638, 2176.9676, 2134.193, 2090.838, 2048.8548, 2007.018, 1966.1742, 1925.4482, 1885.1294, 1846.4776, 1807.4044, 1768.8724, 1731.3732, 1693.4304, 1657.5326, 1621.949, 1586.5532, 1551.7256, 1517.6182, 1483.5186, 1450.4528, 1417.865, 1385.7164, 1352.6828, 1322.6708, 1291.8312, 1260.9036, 1231.476, 1201.8652, 1173.6718, 1145.757, 1119.2072, 1092.2828, 1065.0434, 1038.6264, 1014.3192, 988.5746, 965.0816, 940.1176, 917.9796, 894.5576, 871.1858, 849.9144, 827.1142, 805.0818, 783.9664, 763.9096, 742.0816, 724.3962, 706.3454, 688.018, 667.4214, 650.3106, 633.0686, 613.8094, 597.818, 581.4248, 563.834, 547.363, 531.5066, 520.455400000001, 505.583199999999, 488.366, 476.480799999999, 459.7682, 450.0522, 434.328799999999, 423.952799999999, 408.727000000001, 399.079400000001, 387.252200000001, 373.987999999999, 360.852000000001, 351.6394, 339.642, 330.902400000001, 322.661599999999, 311.662200000001, 301.3254, 291.7484, 279.939200000001, 276.7508, 263.215200000001, 254.811400000001, 245.5494, 242.306399999999, 234.8734, 223.787200000001, 217.7156, 212.0196, 200.793, 195.9748, 189.0702, 182.449199999999, 177.2772, 170.2336, 164.741, 158.613600000001, 155.311, 147.5964, 142.837, 137.3724, 132.0162, 130.0424, 121.9804, 120.451800000001, 114.8968, 111.585999999999, 105.933199999999, 101.705, 98.5141999999996, 95.0488000000005, 89.7880000000005, 91.4750000000004, 83.7764000000006, 80.9698000000008, 72.8574000000008, 73.1615999999995, 67.5838000000003, 62.6263999999992, 63.2638000000006, 66.0977999999996, 52.0843999999997, 58.9956000000002, 47.0912000000008, 46.4956000000002, 48.4383999999991, 47.1082000000006, 43.2392, 37.2759999999998, 40.0283999999992, 35.1864000000005, 35.8595999999998, 32.0998, 28.027, 23.6694000000007, 33.8266000000003, 26.3736000000008, 27.2008000000005, 21.3245999999999, 26.4115999999995, 23.4521999999997, 19.5013999999992, 19.8513999999996, 10.7492000000002, 18.6424000000006, 13.1265999999996, 18.2436000000016, 6.71860000000015, 3.39459999999963, 6.33759999999893, 7.76719999999841, 0.813999999998487, 3.82819999999992, 0.826199999999517, 8.07440000000133, -1.59080000000176, 5.01780000000144, 0.455399999998917, -0.24199999999837, 0.174800000000687, -9.07640000000174, -4.20160000000033, -3.77520000000004, -4.75179999999818, -5.3724000000002, -8.90680000000066, -6.10239999999976, -5.74120000000039, -9.95339999999851, -3.86339999999836, -13.7304000000004, -16.2710000000006, -7.51359999999841, -3.30679999999847, -13.1339999999982, -10.0551999999989, -6.72019999999975, -8.59660000000076, -10.9307999999983, -1.8775999999998, -4.82259999999951, -13.7788, -21.6470000000008, -10.6735999999983, -15.7799999999988}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p13[] = {5907.5052, 5802.2672, 5697.347, 5593.5794, 5491.2622, 5390.5514, 5290.3376, 5191.6952, 5093.5988, 4997.3552, 4902.5972, 4808.3082, 4715.5646, 4624.109, 4533.8216, 4444.4344, 4356.3802, 4269.2962, 4183.3784, 4098.292, 4014.79, 3932.4574, 3850.6036, 3771.2712, 3691.7708, 3615.099, 3538.1858, 3463.4746, 3388.8496, 3315.6794, 3244.5448, 3173.7516, 3103.3106, 3033.6094, 2966.5642, 2900.794, 2833.7256, 2769.81, 2707.3196, 2644.0778, 2583.9916, 2523.4662, 2464.124, 2406.073, 2347.0362, 2292.1006, 2238.1716, 2182.7514, 2128.4884, 2077.1314, 2025.037, 1975.3756, 1928.933, 1879.311, 1831.0006, 1783.2144, 1738.3096, 1694.5144, 1649.024, 1606.847, 1564.7528, 1525.3168, 1482.5372, 1443.9668, 1406.5074, 1365.867, 1329.2186, 1295.4186, 1257.9716, 1225.339, 1193.2972, 1156.3578, 1125.8686, 1091.187, 1061.4094, 1029.4188, 1000.9126, 972.3272, 944.004199999999, 915.7592, 889.965, 862.834200000001, 840.4254, 812.598399999999, 785.924200000001, 763.050999999999, 741.793799999999, 721.466, 699.040799999999, 677.997200000002, 649.866999999998, 634.911800000002, 609.8694, 591.981599999999, 570.2922, 557.129199999999, 538.3858, 521.872599999999, 502.951400000002, 495.776399999999, 475.171399999999, 459.751, 439.995200000001, 426.708999999999, 413.7016, 402.3868, 387.262599999998, 372.0524, 357.050999999999, 342.5098, 334.849200000001, 322.529399999999, 311.613799999999, 295.848000000002, 289.273000000001, 274.093000000001, 263.329600000001, 251.389599999999, 245.7392, 231.9614, 229.7952, 217.155200000001, 208.9588, 199.016599999999, 190.839199999999, 180.6976, 176.272799999999, 166.976999999999, 162.5252, 151.196400000001, 149.386999999999, 133.981199999998, 130.0586, 130.164000000001, 122.053400000001, 110.7428, 108.1276, 106.232400000001, 100.381600000001, 98.7668000000012, 86.6440000000002, 79.9768000000004, 82.4722000000002, 68.7026000000005, 70.1186000000016, 71.9948000000004, 58.998599999999, 59.0492000000013, 56.9818000000014, 47.5338000000011, 42.9928, 51.1591999999982, 37.2740000000013, 42.7220000000016, 31.3734000000004, 26.8090000000011, 25.8934000000008, 26.5286000000015, 29.5442000000003, 19.3503999999994, 26.0760000000009, 17.9527999999991, 14.8419999999969, 10.4683999999979, 8.65899999999965, 9.86720000000059, 4.34139999999752, -0.907800000000861, -3.32080000000133, -0.936199999996461, -11.9916000000012, -8.87000000000262, -6.33099999999831, -11.3366000000024, -15.9207999999999, -9.34659999999712, -15.5034000000014, -19.2097999999969, -15.357799999998, -28.2235999999975, -30.6898000000001, -19.3271999999997, -25.6083999999973, -24.409599999999, -13.6385999999984, -33.4473999999973, -32.6949999999997, -28.9063999999998, -31.7483999999968, -32.2935999999972, -35.8329999999987, -47.620600000002, -39.0855999999985, -33.1434000000008, -46.1371999999974, -37.5892000000022, -46.8164000000033, -47.3142000000007, -60.2914000000019, -37.7575999999972}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p14[] = {11816.475, 11605.0046, 11395.3792, 11188.7504, 10984.1814, 10782.0086, 10582.0072, 10384.503, 10189.178, 9996.2738, 9806.0344, 9617.9798, 9431.394, 9248.7784, 9067.6894, 8889.6824, 8712.9134, 8538.8624, 8368.4944, 8197.7956, 8031.8916, 7866.6316, 7703.733, 7544.5726, 7386.204, 7230.666, 7077.8516, 6926.7886, 6778.6902, 6631.9632, 6487.304, 6346.7486, 6206.4408, 6070.202, 5935.2576, 5799.924, 5671.0324, 5541.9788, 5414.6112, 5290.0274, 5166.723, 5047.6906, 4929.162, 4815.1406, 4699.127, 4588.5606, 4477.7394, 4369.4014, 4264.2728, 4155.9224, 4055.581, 3955.505, 3856.9618, 3761.3828, 3666.9702, 3575.7764, 3482.4132, 3395.0186, 3305.8852, 3221.415, 3138.6024, 3056.296, 2970.4494, 2896.1526, 2816.8008, 2740.2156, 2670.497, 2594.1458, 2527.111, 2460.8168, 2387.5114, 2322.9498, 2260.6752, 2194.2686, 2133.7792, 2074.767, 2015.204, 1959.4226, 1898.6502, 1850.006, 1792.849, 1741.4838, 1687.9778, 1638.1322, 1589.3266, 1543.1394, 1496.8266, 1447.8516, 1402.7354, 1361.9606, 1327.0692, 1285.4106, 1241.8112, 1201.6726, 1161.973, 1130.261, 1094.2036, 1048.2036, 1020.6436, 990.901400000002, 961.199800000002, 924.769800000002, 899.526400000002, 872.346400000002, 834.375, 810.432000000001, 780.659800000001, 756.013800000001, 733.479399999997, 707.923999999999, 673.858, 652.222399999999, 636.572399999997, 615.738599999997, 586.696400000001, 564.147199999999, 541.679600000003, 523.943599999999, 505.714599999999, 475.729599999999, 461.779600000002, 449.750800000002, 439.020799999998, 412.7886, 400.245600000002, 383.188199999997, 362.079599999997, 357.533799999997, 334.319000000003, 327.553399999997, 308.559399999998, 291.270199999999, 279.351999999999, 271.791400000002, 252.576999999997, 247.482400000001, 236.174800000001, 218.774599999997, 220.155200000001, 208.794399999999, 201.223599999998, 182.995600000002, 185.5268, 164.547400000003, 176.5962, 150.689599999998, 157.8004, 138.378799999999, 134.021200000003, 117.614399999999, 108.194000000003, 97.0696000000025, 89.6042000000016, 95.6030000000028, 84.7810000000027, 72.635000000002, 77.3482000000004, 59.4907999999996, 55.5875999999989, 50.7346000000034, 61.3916000000027, 50.9149999999936, 39.0384000000049, 58.9395999999979, 29.633600000001, 28.2032000000036, 26.0078000000067, 17.0387999999948, 9.22000000000116, 13.8387999999977, 8.07240000000456, 14.1549999999988, 15.3570000000036, 3.42660000000615, 6.24820000000182, -2.96940000000177, -8.79940000000352, -5.97860000000219, -14.4048000000039, -3.4143999999942, -13.0148000000045, -11.6977999999945, -25.7878000000055, -22.3185999999987, -24.409599999999, -31.9756000000052, -18.9722000000038, -22.8678000000073, -30.8972000000067, -32.3715999999986, -22.3907999999938, -43.6720000000059, -35.9038, -39.7492000000057, -54.1641999999993, -45.2749999999942, -42.2989999999991, -44.1089999999967, -64.3564000000042, -49.9551999999967, -42.6116000000038}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p15[] = {23634.0036, 23210.8034, 22792.4744, 22379.1524, 21969.7928, 21565.326, 21165.3532, 20770.2806, 20379.9892, 19994.7098, 19613.318, 19236.799, 18865.4382, 18498.8244, 18136.5138, 17778.8668, 17426.2344, 17079.32, 16734.778, 16397.2418, 16063.3324, 15734.0232, 15409.731, 15088.728, 14772.9896, 14464.1402, 14157.5588, 13855.5958, 13559.3296, 13264.9096, 12978.326, 12692.0826, 12413.8816, 12137.3192, 11870.2326, 11602.5554, 11340.3142, 11079.613, 10829.5908, 10583.5466, 10334.0344, 10095.5072, 9859.694, 9625.2822, 9395.7862, 9174.0586, 8957.3164, 8738.064, 8524.155, 8313.7396, 8116.9168, 7913.542, 7718.4778, 7521.65, 7335.5596, 7154.2906, 6968.7396, 6786.3996, 6613.236, 6437.406, 6270.6598, 6107.7958, 5945.7174, 5787.6784, 5635.5784, 5482.308, 5337.9784, 5190.0864, 5045.9158, 4919.1386, 4771.817, 4645.7742, 4518.4774, 4385.5454, 4262.6622, 4142.74679999999, 4015.5318, 3897.9276, 3790.7764, 3685.13800000001, 3573.6274, 3467.9706, 3368.61079999999, 3271.5202, 3170.3848, 3076.4656, 2982.38400000001, 2888.4664, 2806.4868, 2711.9564, 2634.1434, 2551.3204, 2469.7662, 2396.61139999999, 2318.9902, 2243.8658, 2171.9246, 2105.01360000001, 2028.8536, 1960.9952, 1901.4096, 1841.86079999999, 1777.54700000001, 1714.5802, 1654.65059999999, 1596.311, 1546.2016, 1492.3296, 1433.8974, 1383.84600000001, 1339.4152, 1293.5518, 1245.8686, 1193.50659999999, 1162.27959999999, 1107.19439999999, 1069.18060000001, 1035.09179999999, 999.679000000004, 957.679999999993, 925.300199999998, 888.099400000006, 848.638600000006, 818.156400000007, 796.748399999997, 752.139200000005, 725.271200000003, 692.216, 671.633600000001, 647.939799999993, 621.670599999998, 575.398799999995, 561.226599999995, 532.237999999998, 521.787599999996, 483.095799999996, 467.049599999998, 465.286399999997, 415.548599999995, 401.047399999996, 380.607999999993, 377.362599999993, 347.258799999996, 338.371599999999, 310.096999999994, 301.409199999995, 276.280799999993, 265.586800000005, 258.994399999996, 223.915999999997, 215.925399999993, 213.503800000006, 191.045400000003, 166.718200000003, 166.259000000005, 162.941200000001, 148.829400000002, 141.645999999993, 123.535399999993, 122.329800000007, 89.473399999988, 80.1962000000058, 77.5457999999926, 59.1056000000099, 83.3509999999951, 52.2906000000075, 36.3979999999865, 40.6558000000077, 42.0003999999899, 19.6630000000005, 19.7153999999864, -8.38539999999921, -0.692799999989802, 0.854800000000978, 3.23219999999856, -3.89040000000386, -5.25880000001052, -24.9052000000083, -22.6837999999989, -26.4286000000138, -34.997000000003, -37.0216000000073, -43.430400000012, -58.2390000000014, -68.8034000000043, -56.9245999999985, -57.8583999999973, -77.3097999999882, -73.2793999999994, -81.0738000000129, -87.4530000000086, -65.0254000000132, -57.296399999992, -96.2746000000043, -103.25, -96.081600000005, -91.5542000000132, -102.465200000006, -107.688599999994, -101.458000000013, -109.715800000005}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p16[] = {47270.0, 46423.3584, 45585.7074, 44757.152, 43938.8416, 43130.9514, 42330.03, 41540.407, 40759.6348, 39988.206, 39226.5144, 38473.2096, 37729.795, 36997.268, 36272.6448, 35558.665, 34853.0248, 34157.4472, 33470.5204, 32793.5742, 32127.0194, 31469.4182, 30817.6136, 30178.6968, 29546.8908, 28922.8544, 28312.271, 27707.0924, 27114.0326, 26526.692, 25948.6336, 25383.7826, 24823.5998, 24272.2974, 23732.2572, 23201.4976, 22674.2796, 22163.6336, 21656.515, 21161.7362, 20669.9368, 20189.4424, 19717.3358, 19256.3744, 18795.9638, 18352.197, 17908.5738, 17474.391, 17052.918, 16637.2236, 16228.4602, 15823.3474, 15428.6974, 15043.0284, 14667.6278, 14297.4588, 13935.2882, 13578.5402, 13234.6032, 12882.1578, 12548.0728, 12219.231, 11898.0072, 11587.2626, 11279.9072, 10973.5048, 10678.5186, 10392.4876, 10105.2556, 9825.766, 9562.5444, 9294.2222, 9038.2352, 8784.848, 8533.2644, 8301.7776, 8058.30859999999, 7822.94579999999, 7599.11319999999, 7366.90779999999, 7161.217, 6957.53080000001, 6736.212, 6548.21220000001, 6343.06839999999, 6156.28719999999, 5975.15419999999, 5791.75719999999, 5621.32019999999, 5451.66, 5287.61040000001, 5118.09479999999, 4957.288, 4798.4246, 4662.17559999999, 4512.05900000001, 4364.68539999999, 4220.77720000001, 4082.67259999999, 3957.19519999999, 3842.15779999999, 3699.3328, 3583.01180000001, 3473.8964, 3338.66639999999, 3233.55559999999, 3117.799, 3008.111, 2909.69140000001, 2814.86499999999, 2719.46119999999, 2624.742, 2532.46979999999, 2444.7886, 2370.1868, 2272.45259999999, 2196.19260000001, 2117.90419999999, 2023.2972, 1969.76819999999, 1885.58979999999, 1833.2824, 1733.91200000001, 1682.54920000001, 1604.57980000001, 1556.11240000001, 1491.3064, 1421.71960000001, 1371.22899999999, 1322.1324, 1264.7892, 1196.23920000001, 1143.8474, 1088.67240000001, 1073.60380000001, 1023.11660000001, 959.036400000012, 927.433199999999, 906.792799999996, 853.433599999989, 841.873800000001, 791.1054, 756.899999999994, 704.343200000003, 672.495599999995, 622.790399999998, 611.254799999995, 567.283200000005, 519.406599999988, 519.188400000014, 495.312800000014, 451.350799999986, 443.973399999988, 431.882199999993, 392.027000000002, 380.924200000009, 345.128999999986, 298.901400000002, 287.771999999997, 272.625, 247.253000000026, 222.490600000019, 223.590000000026, 196.407599999977, 176.425999999978, 134.725199999986, 132.4804, 110.445599999977, 86.7939999999944, 56.7038000000175, 64.915399999998, 38.3726000000024, 37.1606000000029, 46.170999999973, 49.1716000000015, 15.3362000000197, 6.71639999997569, -34.8185999999987, -39.4476000000141, 12.6830000000191, -12.3331999999937, -50.6565999999875, -59.9538000000175, -65.1054000000004, -70.7576000000117, -106.325200000021, -126.852200000023, -110.227599999984, -132.885999999999, -113.897200000007, -142.713800000027, -151.145399999979, -150.799200000009, -177.756200000003, -156.036399999983, -182.735199999996, -177.259399999981, -198.663600000029, -174.577600000019, -193.84580000001}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p17[] = {94541.0, 92848.811, 91174.019, 89517.558, 87879.9705, 86262.7565, 84663.5125, 83083.7435, 81521.7865, 79977.272, 78455.9465, 76950.219, 75465.432, 73994.152, 72546.71, 71115.2345, 69705.6765, 68314.937, 66944.2705, 65591.255, 64252.9485, 62938.016, 61636.8225, 60355.592, 59092.789, 57850.568, 56624.518, 55417.343, 54231.1415, 53067.387, 51903.526, 50774.649, 49657.6415, 48561.05, 47475.7575, 46410.159, 45364.852, 44327.053, 43318.4005, 42325.6165, 41348.4595, 40383.6265, 39436.77, 38509.502, 37594.035, 36695.939, 35818.6895, 34955.691, 34115.8095, 33293.949, 32465.0775, 31657.6715, 30877.2585, 30093.78, 29351.3695, 28594.1365, 27872.115, 27168.7465, 26477.076, 25774.541, 25106.5375, 24452.5135, 23815.5125, 23174.0655, 22555.2685, 21960.2065, 21376.3555, 20785.1925, 20211.517, 19657.0725, 19141.6865, 18579.737, 18081.3955, 17578.995, 17073.44, 16608.335, 16119.911, 15651.266, 15194.583, 14749.0495, 14343.4835, 13925.639, 13504.509, 13099.3885, 12691.2855, 12328.018, 11969.0345, 11596.5145, 11245.6355, 10917.6575, 10580.9785, 10277.8605, 9926.58100000001, 9605.538, 9300.42950000003, 8989.97850000003, 8728.73249999998, 8448.3235, 8175.31050000002, 7898.98700000002, 7629.79100000003, 7413.76199999999, 7149.92300000001, 6921.12650000001, 6677.1545, 6443.28000000003, 6278.23450000002, 6014.20049999998, 5791.20299999998, 5605.78450000001, 5438.48800000001, 5234.2255, 5059.6825, 4887.43349999998, 4682.935, 4496.31099999999, 4322.52250000002, 4191.42499999999, 4021.24200000003, 3900.64799999999, 3762.84250000003, 3609.98050000001, 3502.29599999997, 3363.84250000003, 3206.54849999998, 3079.70000000001, 2971.42300000001, 2867.80349999998, 2727.08100000001, 2630.74900000001, 2496.6165, 2440.902, 2356.19150000002, 2235.58199999999, 2120.54149999999, 2012.25449999998, 1933.35600000003, 1820.93099999998, 1761.54800000001, 1663.09350000002, 1578.84600000002, 1509.48149999999, 1427.3345, 1379.56150000001, 1306.68099999998, 1212.63449999999, 1084.17300000001, 1124.16450000001, 1060.69949999999, 1007.48849999998, 941.194499999983, 879.880500000028, 836.007500000007, 782.802000000025, 748.385499999975, 647.991500000004, 626.730500000005, 570.776000000013, 484.000500000024, 513.98550000001, 418.985499999952, 386.996999999974, 370.026500000036, 355.496999999974, 356.731499999994, 255.92200000002, 259.094000000041, 205.434499999974, 165.374500000034, 197.347500000033, 95.718499999959, 67.6165000000037, 54.6970000000438, 31.7395000000251, -15.8784999999916, 8.42500000004657, -26.3754999999655, -118.425500000012, -66.6629999999423, -42.9745000000112, -107.364999999991, -189.839000000036, -162.611499999999, -164.964999999967, -189.079999999958, -223.931499999948, -235.329999999958, -269.639500000048, -249.087999999989, -206.475499999942, -283.04449999996, -290.667000000016, -304.561499999953, -336.784499999951, -380.386500000022, -283.280499999993, -364.533000000054, -389.059499999974, -364.454000000027, -415.748000000021, -417.155000000028}; -_CUDAX_CUCO_HLL_TUNING_ARR_DECL __bias_data_p18[] = {189083.0, 185696.913, 182348.774, 179035.946, 175762.762, 172526.444, 169329.754, 166166.099, 163043.269, 159958.91, 156907.912, 153906.845, 150924.199, 147996.568, 145093.457, 142239.233, 139421.475, 136632.27, 133889.588, 131174.2, 128511.619, 125868.621, 123265.385, 120721.061, 118181.769, 115709.456, 113252.446, 110840.198, 108465.099, 106126.164, 103823.469, 101556.618, 99308.004, 97124.508, 94937.803, 92833.731, 90745.061, 88677.627, 86617.47, 84650.442, 82697.833, 80769.132, 78879.629, 77014.432, 75215.626, 73384.587, 71652.482, 69895.93, 68209.301, 66553.669, 64921.981, 63310.323, 61742.115, 60205.018, 58698.658, 57190.657, 55760.865, 54331.169, 52908.167, 51550.273, 50225.254, 48922.421, 47614.533, 46362.049, 45098.569, 43926.083, 42736.03, 41593.473, 40425.26, 39316.237, 38243.651, 37170.617, 36114.609, 35084.19, 34117.233, 33206.509, 32231.505, 31318.728, 30403.404, 29540.0550000001, 28679.236, 27825.862, 26965.216, 26179.148, 25462.08, 24645.952, 23922.523, 23198.144, 22529.128, 21762.4179999999, 21134.779, 20459.117, 19840.818, 19187.04, 18636.3689999999, 17982.831, 17439.7389999999, 16874.547, 16358.2169999999, 15835.684, 15352.914, 14823.681, 14329.313, 13816.897, 13342.874, 12880.882, 12491.648, 12021.254, 11625.392, 11293.7610000001, 10813.697, 10456.209, 10099.074, 9755.39000000001, 9393.18500000006, 9047.57900000003, 8657.98499999999, 8395.85900000005, 8033.0, 7736.95900000003, 7430.59699999995, 7258.47699999996, 6924.58200000005, 6691.29399999999, 6357.92500000005, 6202.05700000003, 5921.19700000004, 5628.28399999999, 5404.96799999999, 5226.71100000001, 4990.75600000005, 4799.77399999998, 4622.93099999998, 4472.478, 4171.78700000001, 3957.46299999999, 3868.95200000005, 3691.14300000004, 3474.63100000005, 3341.67200000002, 3109.14000000001, 3071.97400000005, 2796.40399999998, 2756.17799999996, 2611.46999999997, 2471.93000000005, 2382.26399999997, 2209.22400000005, 2142.28399999999, 2013.96100000001, 1911.18999999994, 1818.27099999995, 1668.47900000005, 1519.65800000005, 1469.67599999998, 1367.13800000004, 1248.52899999998, 1181.23600000003, 1022.71900000004, 1088.20700000005, 959.03600000008, 876.095999999903, 791.183999999892, 703.337000000058, 731.949999999953, 586.86400000006, 526.024999999907, 323.004999999888, 320.448000000091, 340.672999999952, 309.638999999966, 216.601999999955, 102.922999999952, 19.2399999999907, -0.114000000059605, -32.6240000000689, -89.3179999999702, -153.497999999905, -64.2970000000205, -143.695999999996, -259.497999999905, -253.017999999924, -213.948000000091, -397.590000000084, -434.006000000052, -403.475000000093, -297.958000000101, -404.317000000039, -528.898999999976, -506.621000000043, -513.205000000075, -479.351000000024, -596.139999999898, -527.016999999993, -664.681000000099, -680.306000000099, -704.050000000047, -850.486000000034, -757.43200000003, -713.308999999892}; - -//! @brief Get bias data array for a given precision -//! -//! @param __precision The precision value (4-18) -//! @return Pointer to the bias data array for the given precision -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const double* __bias_data(::cuda::std::int32_t __precision) noexcept { - switch (__precision) { - case 4: return __bias_data_p4; - case 5: return __bias_data_p5; - case 6: return __bias_data_p6; - case 7: return __bias_data_p7; - case 8: return __bias_data_p8; - case 9: return __bias_data_p9; - case 10: return __bias_data_p10; - case 11: return __bias_data_p11; - case 12: return __bias_data_p12; - case 13: return __bias_data_p13; - case 14: return __bias_data_p14; - case 15: return __bias_data_p15; - case 16: return __bias_data_p16; - case 17: return __bias_data_p17; - case 18: return __bias_data_p18; - default: return nullptr; - } -} - -// clang-format on -} // namespace cuda::experimental::cuco::__hyperloglog_ns - -#include - -#endif // _CUDAX___CUCO_DETAIL_HYPERLOGLOG_TUNING_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/open_addressing/kernels.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/open_addressing/kernels.cuh deleted file mode 100644 index 8a21a064..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/open_addressing/kernels.cuh +++ /dev/null @@ -1,294 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_OPEN_ADDRESSING_KERNELS_CUH -#define _CUDAX___CUCO_DETAIL_OPEN_ADDRESSING_KERNELS_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include - -#include -#include -#include -#include - -#include - -#include - -#include -#include - -#if _CCCL_CUDA_COMPILATION() - -_CCCL_DIAG_PUSH -_CCCL_DIAG_SUPPRESS_GCC("-Wattributes") - -namespace cuda::experimental::cuco::__open_addressing -{ -//! @brief Scalar (cooperative-group size 1) functor inserting `first[i]` when `pred(stencil[i])` holds. -template -struct __insert_if_fn -{ - _InputIt __first; - _StencilIt __stencil; - _Predicate __pred; - _Ref __ref; - - _CCCL_DEVICE_API void operator()(detail::__index_type __idx) - { - if (__pred(*(__stencil + __idx))) - { - __ref.insert(*(__first + __idx)); - } - } -}; - -template -__insert_if_fn(_InputIt, _StencilIt, _Predicate, _Ref) -> __insert_if_fn<_InputIt, _StencilIt, _Predicate, _Ref>; - -//! @brief Scalar (cooperative-group size 1) functor writing `pred(stencil[i]) ? contains(first[i]) : false`. -template -struct __contains_if_fn -{ - _InputIt __first; - _StencilIt __stencil; - _Predicate __pred; - _OutputIt __output_begin; - _Ref __ref; - - _CCCL_DEVICE_API void operator()(detail::__index_type __idx) const - { - *(__output_begin + __idx) = __pred(*(__stencil + __idx)) ? __ref.contains(*(__first + __idx)) : false; - } -}; - -template -__contains_if_fn(_InputIt, _StencilIt, _Predicate, _OutputIt, _Ref) - -> __contains_if_fn<_InputIt, _StencilIt, _Predicate, _OutputIt, _Ref>; - -//! @brief Inserts all elements in the range `[first, first + n)` and returns the number of -//! successful insertions if `pred` of the corresponding stencil returns true. -template -_CCCL_KERNEL_ATTRIBUTES _CCCL_LAUNCH_BOUNDS(_BlockSize) void __insert_if_n( - _InputIt __first, - detail::__index_type __n, - _StencilIt __stencil, - _Predicate __pred, - typename _Ref::size_type* __num_successes, - _Ref __ref) -{ - using __block_reduce = CUB_NS_QUALIFIER::BlockReduce; - __shared__ typename __block_reduce::TempStorage __temp_storage; - typename _Ref::size_type __thread_num_successes = 0; - - const auto __loop_stride = detail::__grid_stride() / _CgSize; - auto __idx = detail::__global_thread_id() / _CgSize; - - while (__idx < __n) - { - if (__pred(*(__stencil + __idx))) - { - using __value_t = typename ::cuda::std::iterator_traits<_InputIt>::value_type; - const __value_t __insert_element{*(__first + __idx)}; - if constexpr (_CgSize == 1) - { - if (__ref.insert(__insert_element)) - { - __thread_num_successes++; - } - } - else - { - const auto __tile = ::cooperative_groups::tiled_partition<_CgSize, ::cooperative_groups::thread_block>( - ::cooperative_groups::this_thread_block()); - if (__ref.insert(__tile, __insert_element) && __tile.thread_rank() == 0) - { - __thread_num_successes++; - } - } - } - __idx += __loop_stride; - } - - const auto __block_num_successes = __block_reduce(__temp_storage).Sum(__thread_num_successes); - if (threadIdx.x == 0) - { - ::cuda::atomic_ref{*__num_successes}.fetch_add( - __block_num_successes, ::cuda::std::memory_order_relaxed); - } -} - -//! @brief Inserts all elements in the range `[first, first + n)` if `pred` of the corresponding -//! stencil returns true. -template -_CCCL_KERNEL_ATTRIBUTES _CCCL_LAUNCH_BOUNDS(_BlockSize) void -__insert_if_n(_InputIt __first, detail::__index_type __n, _StencilIt __stencil, _Predicate __pred, _Ref __ref) -{ - const auto __loop_stride = detail::__grid_stride() / _CgSize; - auto __idx = detail::__global_thread_id() / _CgSize; - - while (__idx < __n) - { - if (__pred(*(__stencil + __idx))) - { - using __value_t = typename ::cuda::std::iterator_traits<_InputIt>::value_type; - const __value_t __insert_element{*(__first + __idx)}; - const auto __tile = ::cooperative_groups::tiled_partition<_CgSize, ::cooperative_groups::thread_block>( - ::cooperative_groups::this_thread_block()); - __ref.insert(__tile, __insert_element); - } - __idx += __loop_stride; - } -} - -//! @brief Contains test with predicate. -template -_CCCL_KERNEL_ATTRIBUTES _CCCL_LAUNCH_BOUNDS(_BlockSize) void __contains_if_n( - _InputIt __first, - detail::__index_type __n, - _StencilIt __stencil, - _Predicate __pred, - _OutputIt __output_begin, - _Ref __ref) -{ - const auto __block = ::cooperative_groups::this_thread_block(); - const auto __loop_stride = detail::__grid_stride() / _CgSize; - auto __idx = detail::__global_thread_id() / _CgSize; - - while (__idx < __n) - { - const auto __tile = ::cooperative_groups::tiled_partition<_CgSize, ::cooperative_groups::thread_block>(__block); - using __value_t = typename ::cuda::std::iterator_traits<_InputIt>::value_type; - const __value_t __key = *(__first + __idx); - const auto __found = __pred(*(__stencil + __idx)) ? __ref.contains(__tile, __key) : false; - if (__tile.thread_rank() == 0) - { - *(__output_begin + __idx) = __found; - } - __idx += __loop_stride; - } -} - -//! @brief Helper to determine the buffer type for the find kernel. -template -struct __find_buffer -{ - using type = typename _Container::key_type; -}; - -//! @brief Helper to determine the buffer type for the find kernel when `mapped_type` exists. -template -struct __find_buffer<_Container, ::cuda::std::void_t> -{ - using type = typename _Container::mapped_type; -}; - -//! @brief Converts a find result to the output value or the appropriate empty sentinel. -template -[[nodiscard]] _CCCL_DEVICE_API typename __find_buffer<_Ref>::type __find_output(_Ref const& __ref, _Iterator __found) -{ - constexpr bool __has_payload = !::cuda::std::is_same_v; - - if constexpr (__has_payload) - { - return __found == __ref.end() ? __ref.empty_value_sentinel() : __found->second; - } - else - { - return __found == __ref.end() ? __ref.empty_key_sentinel() : *__found; - } -} - -//! @brief Find with predicate. -template -_CCCL_KERNEL_ATTRIBUTES _CCCL_LAUNCH_BOUNDS(_BlockSize) void __find_if_n( - _InputIt __first, - detail::__index_type __n, - _StencilIt __stencil, - _Predicate __pred, - _OutputIt __output_begin, - _Ref __ref) -{ - const auto __block = ::cooperative_groups::this_thread_block(); - const auto __thread_idx = __block.thread_rank(); - const auto __loop_stride = detail::__grid_stride() / _CgSize; - auto __idx = detail::__global_thread_id() / _CgSize; - - using __output_type = typename __find_buffer<_Ref>::type; - __shared__ __output_type __output_buffer[_BlockSize / _CgSize]; - - while ((__idx - __thread_idx / _CgSize) < __n) - { - if constexpr (_CgSize == 1) - { - if (__idx < __n) - { - using __value_t = typename ::cuda::std::iterator_traits<_InputIt>::value_type; - const __value_t __key = *(__first + __idx); - const auto __selected = __pred(*(__stencil + __idx)); - const auto __found = __selected ? __ref.find(__key) : __ref.end(); - /* - * The ld.relaxed.gpu instruction causes L1 to flush more frequently, causing increased - * sector stores from L2 to global memory. By writing results to shared memory and then - * synchronizing before writing back to global, we no longer rely on L1, preventing the - * increase in sector stores from L2 to global and improving performance. - */ - __output_buffer[__thread_idx] = __find_output(__ref, __found); - } - __block.sync(); - if (__idx < __n) - { - *(__output_begin + __idx) = __output_buffer[__thread_idx]; - } - } - else - { - const auto __tile = ::cooperative_groups::tiled_partition<_CgSize, ::cooperative_groups::thread_block>(__block); - if (__idx < __n) - { - using __value_t = typename ::cuda::std::iterator_traits<_InputIt>::value_type; - const __value_t __key = *(__first + __idx); - - bool __selected = false; - if (__tile.thread_rank() == 0) - { - __selected = __pred(*(__stencil + __idx)); - } - __selected = __tile.shfl(__selected, 0); - const auto __found = __selected ? __ref.find(__tile, __key) : __ref.end(); - - if (__tile.thread_rank() == 0) - { - *(__output_begin + __idx) = __find_output(__ref, __found); - } - } - } - __idx += __loop_stride; - } -} -} // namespace cuda::experimental::cuco::__open_addressing - -_CCCL_DIAG_POP - -#endif // _CCCL_CUDA_COMPILATION() - -#include - -#endif // _CUDAX___CUCO_DETAIL_OPEN_ADDRESSING_KERNELS_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/open_addressing/open_addressing_impl.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/open_addressing/open_addressing_impl.cuh deleted file mode 100644 index 661f2756..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/open_addressing/open_addressing_impl.cuh +++ /dev/null @@ -1,426 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_OPEN_ADDRESSING_IMPL_CUH -#define _CUDAX___CUCO_DETAIL_OPEN_ADDRESSING_IMPL_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include - -#if !_CCCL_COMPILER(NVRTC) - -namespace cuda::experimental::cuco::__open_addressing -{ -//! @brief Open addressing implementation class. -//! -//! @note This class should NOT be used directly. -//! -//! @throw If the size of the given key type is larger than 8 bytes -//! @throw If the size of the given slot type is larger than 16 bytes -//! @throw If the given key type doesn't have unique object representations, i.e., -//! `cuda::is_bitwise_comparable_v<_Key> == false` -//! @throw If the probing scheme type is not inherited from -//! `cuda::experimental::cuco::detail::__probing_scheme_base` -//! -//! @tparam _Key Type used for keys. Requires `cuda::is_bitwise_comparable_v<_Key>` -//! @tparam _Value Type used for storage values -//! @tparam _Scope The scope in which operations will be performed by individual threads -//! @tparam _KeyEqual Binary callable type used to compare two keys for equality -//! @tparam _ProbingScheme Probing scheme type -//! @tparam _BucketSize Number of slots per bucket -//! @tparam _MemoryResource Type of memory resource used for device storage -template -class __open_addressing_impl -{ -public: - using __key_type = _Key; - using __value_type = _Value; - using __probing_scheme_type = _ProbingScheme; - using __hasher = typename __probing_scheme_type::hasher; - using __size_type = ::cuda::std::size_t; - using __key_equal = _KeyEqual; - using __storage_ref_type = __slot_storage_ref<__value_type, _BucketSize>; - - static constexpr auto __has_payload = !::cuda::std::is_same_v<_Key, _Value>; - static constexpr auto __cg_size = _ProbingScheme::cg_size; - static constexpr auto __bucket_size = _BucketSize; - static constexpr auto __thread_scope = _Scope; - - static_assert(sizeof(_Key) <= 8, "Container does not support key types larger than 8 bytes."); - static_assert(sizeof(_Value) <= 16, "Container does not support slot types larger than 16 bytes."); - static_assert(::cuda::is_bitwise_comparable_v<_Key>, - "Key type must have unique object representations or have been explicitly declared as safe for " - "bitwise comparison via specialization of cuda::is_bitwise_comparable_v."); - static_assert(::cuda::std::is_base_of_v, _ProbingScheme>, - "ProbingScheme must inherit from cuda::experimental::cuco::detail::__probing_scheme_base"); - -private: - __value_type __empty_slot_sentinel; - __key_type __erased_key_sentinel; - __key_equal __predicate; - __probing_scheme_type __probing_scheme; - mutable _MemoryResource __memory_resource; - ::cuda::device_buffer<__value_type> __slots; - - //! @brief Computes the number of buckets for a requested capacity. - [[nodiscard]] _CCCL_HOST_API static __size_type __compute_num_buckets(__size_type __requested_capacity) - { - return make_valid_capacity<_ProbingScheme, _BucketSize>(__requested_capacity) / _BucketSize; - } - - //! @brief Computes the number of buckets for a given number of keys and load factor. - [[nodiscard]] _CCCL_HOST_API static __size_type __compute_num_buckets(__size_type __n, double __load_factor) - { - return make_valid_capacity<_ProbingScheme, _BucketSize>(__n, __load_factor) / _BucketSize; - } - - //! @brief Extracts the key from a slot. - [[nodiscard]] _CCCL_HOST_API constexpr const __key_type& __extract_key(const __value_type& __slot) const noexcept - { - if constexpr (__has_payload) - { - return __slot.first; - } - else - { - return __slot; - } - } - - //! @brief Allocates and zero-initializes an RAII device counter. - [[nodiscard]] _CCCL_HOST_API ::cuda::device_buffer<__size_type> __make_counter(::cuda::stream_ref __stream) const - { - return ::cuda::device_buffer<__size_type>{__stream, __memory_resource, {__size_type{0}}}; - } - - //! @brief Reads a device counter to host. - [[nodiscard]] _CCCL_HOST_API __size_type - __read_counter(const ::cuda::device_buffer<__size_type>& __counter, ::cuda::stream_ref __stream) const - { - __size_type __result; - ::cuda::__driver::__memcpyAsync(&__result, __counter.data(), sizeof(__size_type), __stream.get()); - __stream.sync(); - return __result; - } - -public: - //! @brief Constructs an open addressing implementation with the given capacity. - _CCCL_HOST_API __open_addressing_impl( - ::cuda::stream_ref __stream, - _MemoryResource __mr, - __size_type __capacity, - __value_type __empty_slot_sentinel, - const _KeyEqual& __pred, - const _ProbingScheme& __probing_scheme) - : __empty_slot_sentinel{__empty_slot_sentinel} - , __erased_key_sentinel{__extract_key(__empty_slot_sentinel)} - , __predicate{__pred} - , __probing_scheme{__probing_scheme} - , __memory_resource{__mr} - , __slots{__stream, __mr, __compute_num_buckets(__capacity) * _BucketSize, ::cuda::no_init} - { - clear_async(__stream); - } - - //! @brief Constructs an open addressing implementation with capacity derived from desired load - //! factor. - _CCCL_HOST_API __open_addressing_impl( - ::cuda::stream_ref __stream, - _MemoryResource __mr, - __size_type __n, - double __desired_load_factor, - __value_type __empty_slot_sentinel, - const _KeyEqual& __pred, - const _ProbingScheme& __probing_scheme) - : __empty_slot_sentinel{__empty_slot_sentinel} - , __erased_key_sentinel{__extract_key(__empty_slot_sentinel)} - , __predicate{__pred} - , __probing_scheme{__probing_scheme} - , __memory_resource{__mr} - , __slots{__stream, __mr, __compute_num_buckets(__n, __desired_load_factor) * _BucketSize, ::cuda::no_init} - { - clear_async(__stream); - } - - //! @brief Constructs an open addressing implementation with erasure support. - _CCCL_HOST_API __open_addressing_impl( - ::cuda::stream_ref __stream, - _MemoryResource __mr, - __size_type __capacity, - __value_type __empty_slot_sentinel, - __key_type __erased_key_sentinel, - const _KeyEqual& __pred, - const _ProbingScheme& __probing_scheme) - : __empty_slot_sentinel{__empty_slot_sentinel} - , __erased_key_sentinel{__erased_key_sentinel} - , __predicate{__pred} - , __probing_scheme{__probing_scheme} - , __memory_resource{__mr} - , __slots{__stream, __mr, __compute_num_buckets(__capacity) * _BucketSize, ::cuda::no_init} - { - if (empty_key_sentinel() == erased_key_sentinel()) - { - _CCCL_THROW(::std::invalid_argument, "The empty key sentinel and erased key sentinel cannot be the same value."); - } - clear_async(__stream); - } - - //! @brief Fills all slots with the empty sentinel. - _CCCL_HOST_API void clear(::cuda::stream_ref __stream) - { - clear_async(__stream); - __stream.sync(); - } - - //! @brief Asynchronously fills all slots with the empty sentinel. - //! - //! @throws cuda_error if the clear operation fails to launch - _CCCL_HOST_API void clear_async(::cuda::stream_ref __stream) - { - const auto __n = capacity(); - if (__n == 0) - { - return; - } - _CCCL_TRY_CUDA_API( - CUB_NS_QUALIFIER::DeviceTransform::Fill, - "cuco: failed to clear slot storage", - __slots.data(), - static_cast(__n), - __empty_slot_sentinel, - __stream); - } - - //! @brief Inserts keys in `[first, last)` and returns the number of successful insertions. - template - _CCCL_HOST_API __size_type insert(::cuda::stream_ref __stream, _InputIt __first, _InputIt __last, _Ref __container_ref) - { - const auto __num_keys = detail::__distance(__first, __last); - if (__num_keys == 0) - { - return 0; - } - - auto __counter = __make_counter(__stream); - - const auto __grid_size = detail::__grid_size(__num_keys, __cg_size); - - __open_addressing::__insert_if_n<__cg_size, detail::__default_block_size> - <<(__grid_size), detail::__default_block_size, 0, __stream.get()>>>( - __first, - __num_keys, - ::cuda::constant_iterator{true}, - ::cuda::std::identity{}, - __counter.data(), - __container_ref); - - return __read_counter(__counter, __stream); - } - - //! @brief Asynchronously inserts keys in `[first, last)`. - //! - //! @throws cuda_error if the insert operation fails to launch - template - _CCCL_HOST_API void insert_async(::cuda::stream_ref __stream, _InputIt __first, _InputIt __last, _Ref __container_ref) - { - const auto __num_keys = detail::__distance(__first, __last); - if (__num_keys == 0) - { - return; - } - - if constexpr (__cg_size == 1) - { - __open_addressing::__insert_if_fn __op{ - __first, ::cuda::constant_iterator{true}, ::cuda::std::identity{}, __container_ref}; - _CCCL_TRY_CUDA_API(CUB_NS_QUALIFIER::DeviceFor::Bulk, "cuco: failed to insert keys", __num_keys, __op, __stream); - } - else - { - const auto __grid_size = detail::__grid_size(__num_keys, __cg_size); - - __open_addressing::__insert_if_n<__cg_size, detail::__default_block_size> - <<(__grid_size), detail::__default_block_size, 0, __stream.get()>>>( - __first, __num_keys, ::cuda::constant_iterator{true}, ::cuda::std::identity{}, __container_ref); - } - } - - //! @brief Asynchronously checks if keys in `[first, last)` exist in the container. - //! - //! @throws cuda_error if the query operation fails to launch - template - _CCCL_HOST_API void contains_async( - ::cuda::stream_ref __stream, _InputIt __first, _InputIt __last, _OutputIt __output_begin, _Ref __container_ref) const - { - const auto __num_keys = detail::__distance(__first, __last); - if (__num_keys == 0) - { - return; - } - - if constexpr (__cg_size == 1) - { - __open_addressing::__contains_if_fn __op{ - __first, ::cuda::constant_iterator{true}, ::cuda::std::identity{}, __output_begin, __container_ref}; - _CCCL_TRY_CUDA_API(CUB_NS_QUALIFIER::DeviceFor::Bulk, "cuco: failed to query keys", __num_keys, __op, __stream); - } - else - { - const auto __grid_size = detail::__grid_size(__num_keys, __cg_size); - - __open_addressing::__contains_if_n<__cg_size, detail::__default_block_size> - <<(__grid_size), detail::__default_block_size, 0, __stream.get()>>>( - __first, - __num_keys, - ::cuda::constant_iterator{true}, - ::cuda::std::identity{}, - __output_begin, - __container_ref); - } - } - - //! @brief Asynchronously finds payloads for keys in `[first, last)` whose stencil satisfies `pred`. - //! - //! For each key `first[i]` with `pred(stencil[i]) == true` that is present, the associated payload is - //! written to the corresponding output position; otherwise the empty value sentinel is written. - //! - //! @throws cuda_error if the query operation fails to launch - template - _CCCL_HOST_API void find_if_async( - ::cuda::stream_ref __stream, - _InputIt __first, - _InputIt __last, - _StencilIt __stencil, - _Predicate __pred, - _OutputIt __output_begin, - _Ref __container_ref) const - { - const auto __num_keys = detail::__distance(__first, __last); - if (__num_keys == 0) - { - return; - } - - const auto __grid_size = detail::__grid_size(__num_keys, __cg_size); - - __open_addressing::__find_if_n<__cg_size, detail::__default_block_size> - <<(__grid_size), detail::__default_block_size, 0, __stream.get()>>>( - __first, __num_keys, __stencil, __pred, __output_begin, __container_ref); - } - - //! @brief Asynchronously finds the payloads for keys in `[first, last)`. - //! - //! For each key that is present, the associated payload is written to the corresponding output - //! position; for each key that is absent, the empty value sentinel is written instead. - //! - //! @throws cuda_error if the query operation fails to launch - template - _CCCL_HOST_API void find_async( - ::cuda::stream_ref __stream, _InputIt __first, _InputIt __last, _OutputIt __output_begin, _Ref __container_ref) const - { - this->find_if_async( - __stream, - __first, - __last, - ::cuda::constant_iterator{true}, - ::cuda::std::identity{}, - __output_begin, - __container_ref); - } - - //! @brief Returns the total number of slots. - [[nodiscard]] _CCCL_HOST_API constexpr __size_type capacity() const noexcept - { - return static_cast<__size_type>(__slots.size()); - } - - //! @brief Returns a pointer to the underlying slot array. - [[nodiscard]] _CCCL_HOST_API __value_type* data() const noexcept - { - return const_cast<__value_type*>(__slots.data()); - } - - //! @brief Returns the empty key sentinel. - [[nodiscard]] _CCCL_HOST_API constexpr __key_type empty_key_sentinel() const noexcept - { - return __extract_key(__empty_slot_sentinel); - } - - //! @brief Returns the erased key sentinel. - [[nodiscard]] _CCCL_HOST_API constexpr __key_type erased_key_sentinel() const noexcept - { - return __erased_key_sentinel; - } - - //! @brief Returns the key comparison function. - [[nodiscard]] _CCCL_HOST_API constexpr __key_equal key_eq() const noexcept - { - return __predicate; - } - - //! @brief Returns the probing scheme. - [[nodiscard]] _CCCL_HOST_API constexpr __probing_scheme_type probing_scheme() const noexcept - { - return __probing_scheme; - } - - //! @brief Returns the hash function. - [[nodiscard]] _CCCL_HOST_API constexpr __hasher hash_function() const noexcept - { - return probing_scheme().hash_function(); - } - - //! @brief Returns a non-owning reference to the stored slots. - [[nodiscard]] _CCCL_HOST_API __storage_ref_type storage_ref() const noexcept - { - return __storage_ref_type{const_cast<__value_type*>(__slots.data()), capacity()}; - } -}; -} // namespace cuda::experimental::cuco::__open_addressing - -#endif // !_CCCL_COMPILER(NVRTC) - -#include - -#endif // _CUDAX___CUCO_DETAIL_OPEN_ADDRESSING_IMPL_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/open_addressing/open_addressing_ref_impl.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/open_addressing/open_addressing_ref_impl.cuh deleted file mode 100644 index 5f3ea37a..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/open_addressing/open_addressing_ref_impl.cuh +++ /dev/null @@ -1,1028 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_OPEN_ADDRESSING_REF_IMPL_CUH -#define _CUDAX___CUCO_DETAIL_OPEN_ADDRESSING_REF_IMPL_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include - -#include - -namespace cuda::experimental::cuco::__open_addressing -{ -//! @brief Three-way insert result enum. -enum class __insert_result : ::cuda::std::int8_t -{ - __continue = 0, - __success = 1, - __duplicate = 2 -}; - -//! -//! @brief Helper struct to store intermediate bucket probing results. -struct __bucket_probing_results -{ - detail::__equal_result __state; ///< Equal result - ::cuda::std::int32_t __intra_bucket_index; ///< Intra-bucket index - -#if _CCCL_CUDA_COMPILATION() - //! - //! @brief Constructs __bucket_probing_results. - //! - //! @param __state The three way equality __result - //! @param __index Intra-bucket index - _CCCL_DEVICE_API explicit constexpr __bucket_probing_results( - detail::__equal_result __state, ::cuda::std::int32_t __index) noexcept - : __state{__state} - , __intra_bucket_index{__index} - {} -#endif // _CCCL_CUDA_COMPILATION() -}; - -//! -//! @brief Common device non-owning "ref" implementation class. -//! -//! @note This class should NOT be used directly. -//! -//! @throw If the size of the given __key type is larger than 8 bytes -//! @throw If the given __key type doesn't have unique object representations, i.e., -//! `bitwise_comparable_v == false` -//! @throw If the probing scheme type is not inherited from -//! `detail::__probing_scheme_base` -//! -//! @tparam Key Type used for keys. Requires `::cuda::is_bitwise_comparable_v` returning true -//! @tparam Scope The scope in which operations will be performed by individual threads. -//! @tparam KeyEqual Binary callable type used to compare two keys for equality -//! @tparam ProbingScheme Probing scheme (see `cuda/experimental/__cuco/probing_scheme.cuh` for options) -//! @tparam StorageRef Storage ref type -//! @tparam AllowsDuplicates Flag indicating whether duplicate keys are allowed or not -template -class __open_addressing_ref_impl -{ - static_assert(sizeof(_Key) <= 8, "Container does not support __key types larger than 8 bytes."); - - static_assert(::cuda::is_bitwise_comparable_v<_Key>, - "Key type must have unique object representations or have been explicitly declared as safe for " - "bitwise comparison via specialization of ::cuda::is_bitwise_comparable_v<_Key>."); - - static_assert(::cuda::std::is_base_of_v, _ProbingScheme>, - "ProbingScheme must inherit from ::cuda::experimental::cuco::detail::__probing_scheme_base"); - -public: - using __key_type = _Key; ///< Key type - using __probing_scheme_type = _ProbingScheme; ///< Type of probing scheme - using __hasher = typename __probing_scheme_type::hasher; ///< Hash function type - using __storage_ref_type = _StorageRef; ///< Type of storage ref - using __bucket_type = typename __storage_ref_type::__bucket_type; ///< Bucket type - using __value_type = typename __storage_ref_type::__value_type; ///< Storage element type - using __size_type = typename __storage_ref_type::__size_type; ///< Probing scheme size type - using __key_equal = _KeyEqual; ///< Type of key equality binary callable - using __iterator = typename __storage_ref_type::__iterator; ///< Slot iterator type - using __const_iterator = typename __storage_ref_type::__const_iterator; ///< Const slot iterator type - - static constexpr auto __cg_size = __probing_scheme_type::cg_size; ///< Cooperative group size - static constexpr auto __bucket_size = __storage_ref_type::__bucket_size; ///< Bucket size - static constexpr auto __thread_scope = _Scope; ///< CUDA thread scope - -private: - /// Determines if the container is a key/value or key-only store - static constexpr auto __has_payload = !::cuda::std::is_same_v<_Key, typename _StorageRef::__value_type>; - - static constexpr auto __has_packable_representation = - sizeof(__value_type) <= 8 && ::cuda::is_power_of_two(sizeof(__value_type)) - && ::cuda::std::is_trivially_copyable_v<__value_type>; - - /// Flag indicating whether duplicate keys are allowed or not - static constexpr auto __allows_duplicates = _AllowsDuplicates; - - __value_type __empty_slot_sentinel; ///< Sentinel value indicating an empty slot - detail::__equal_wrapper<__key_type, __key_equal, __allows_duplicates> __predicate; ///< Key equality - __probing_scheme_type __probing_scheme; ///< Probing scheme - __storage_ref_type __storage_ref; ///< Slot storage ref - -public: - //! @brief Constructs `__open_addressing_ref_impl`. - //! - //! @param __empty_slot_sentinel Sentinel indicating an empty slot - //! @param __predicate Key equality binary callable - //! @param __probing_scheme Probing scheme - //! @param __storage_ref Non-owning ref of slot storage - _CCCL_HOST_DEVICE_API explicit constexpr __open_addressing_ref_impl( - __value_type __empty_slot_sentinel, - const __key_equal& __predicate, - const __probing_scheme_type& __probing_scheme, - __storage_ref_type __storage_ref) noexcept - : __empty_slot_sentinel{__empty_slot_sentinel} - , __predicate{__extract_key(__empty_slot_sentinel), __extract_key(__empty_slot_sentinel), __predicate} - , __probing_scheme{__probing_scheme} - , __storage_ref{__storage_ref} - {} - - //! @brief Constructs `__open_addressing_ref_impl`. - //! - //! @param __empty_slot_sentinel Sentinel indicating an empty slot - //! @param __erased_key_sentinel Sentinel indicating an erased __key - //! @param __predicate Key equality binary callable - //! @param __probing_scheme Probing scheme - //! @param __storage_ref Non-owning ref of slot storage - _CCCL_HOST_DEVICE_API explicit constexpr __open_addressing_ref_impl( - __value_type __empty_slot_sentinel, - __key_type __erased_key_sentinel, - const __key_equal& __predicate, - const __probing_scheme_type& __probing_scheme, - __storage_ref_type __storage_ref) noexcept - : __empty_slot_sentinel{__empty_slot_sentinel} - , __predicate{__extract_key(__empty_slot_sentinel), __erased_key_sentinel, __predicate} - , __probing_scheme{__probing_scheme} - , __storage_ref{__storage_ref} - {} - - //! @brief Gets the sentinel value used to represent an empty __key slot. - //! - //! @return The sentinel value used to represent an empty __key slot - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __key_type empty_key_sentinel() const noexcept - { - return __predicate.__empty_sentinel; - } - - //! @brief Gets the sentinel value used to represent an empty payload slot. - //! - //! @return The sentinel value used to represent an empty payload slot - _CCCL_TEMPLATE(bool _Dummy = true) - _CCCL_REQUIRES(__has_payload _CCCL_AND _Dummy) - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto empty_value_sentinel() const noexcept - { - return __extract_payload(empty_slot_sentinel()); - } - - //! @brief Gets the sentinel value used to represent an erased __key slot. - //! - //! @return The sentinel value used to represent an erased __key slot - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __key_type erased_key_sentinel() const noexcept - { - return __predicate.__erased_sentinel; - } - - //! @brief Gets the sentinel used to represent an empty slot. - //! - //! @return The sentinel value used to represent an empty slot - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __value_type empty_slot_sentinel() const noexcept - { - return __empty_slot_sentinel; - } - - //! @brief Returns the function that compares keys for equality. - //! - //! @return The key equality predicate - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr detail::__equal_wrapper<__key_type, __key_equal, __allows_duplicates> - predicate() const noexcept - { - return __predicate; - } - - //! @brief Gets the key comparator. - //! - //! @return The comparator used to compare keys - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __key_equal key_eq() const noexcept - { - return predicate().__equal; - } - - //! @brief Gets the probing scheme. - //! - //! @return The probing scheme used for the container - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __probing_scheme_type probing_scheme() const noexcept - { - return __probing_scheme; - } - - //! @brief Gets the function(s) used to hash keys - //! - //! @return The function(s) used to hash keys - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __hasher hash_function() const noexcept - { - return probing_scheme().hash_function(); - } - - //! @brief Gets the non-owning storage ref. - //! - //! @return The non-owning storage ref of the container - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __storage_ref_type storage_ref() const noexcept - { - return __storage_ref; - } - - //! - //! @brief Gets the maximum number of elements the container can hold. - //! - //! @return The maximum number of elements the container can hold - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto capacity() const noexcept - { - return __storage_ref.capacity(); - } - - //! - //! @brief Returns an iterator to one past the last slot. - //! - //! @return An iterator to one past the last slot - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __iterator end() const noexcept - { - return __storage_ref.end(); - } - - //! - //! @brief Returns an iterator to one past the last slot. - //! - //! @return An iterator to one past the last slot - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __iterator end() noexcept - { - return __storage_ref.end(); - } - -#if _CCCL_CUDA_COMPILATION() - //! - //! @brief Inserts an element. - //! - //! @tparam Value Input type which is convertible to '__value_type' - //! - //! @param __value The element to insert - //! - //! @return True if the given element is successfully inserted - template - _CCCL_DEVICE_API bool insert(_Value __value) noexcept - { - static_assert(__cg_size == 1, "Non-CG operation is incompatible with the current probing scheme"); - - const auto __val = __heterogeneous_value(__value); - const auto __key = __extract_key(__val); - - auto __probing_iter = - __probing_scheme.template make_iterator<__bucket_size>(__key, __storage_ref.capacity_extent()); - const auto __init_idx = *__probing_iter; - - while (true) - { - const auto __bucket_slots = __storage_ref[*__probing_iter]; - - for (auto& __slot_content : __bucket_slots) - { - const auto __eq_res = - __predicate.template operator()(__key, __extract_key(__slot_content)); - - if constexpr (!__allows_duplicates) - { - // If the __key is already in the container, return false - if (__eq_res == detail::__equal_result::__equal) - { - return false; - } - } - if (__eq_res == detail::__equal_result::__available) - { - const auto __intra_bucket_index = &__slot_content - __bucket_slots.data(); - switch (__attempt_insert(__get_slot_ptr(*__probing_iter, __intra_bucket_index), __slot_content, __val)) - { - case __insert_result::__duplicate: { - if constexpr (!__allows_duplicates) - { - return false; - } - [[fallthrough]]; - } - case __insert_result::__continue: - continue; - case __insert_result::__success: - return true; - } - } - } - ++__probing_iter; - if (*__probing_iter == __init_idx) - { - return false; - } - } - } - - //! - //! @brief Inserts an element. - //! - //! @tparam Value Input type which is convertible to '__value_type' - //! @tparam ParentCG Type of parent Cooperative Group - //! - //! @param __group The Cooperative Group used to perform group insert - //! @param __value The element to insert - //! - //! @return True if the given element is successfully inserted - template - _CCCL_DEVICE_API bool - insert(::cooperative_groups::thread_block_tile<__cg_size, _ParentCG> __group, _Value __value) noexcept - { - const auto __val = __heterogeneous_value(__value); - const auto __key = __extract_key(__val); - auto __probing_iter = - __probing_scheme.template make_iterator<__bucket_size>(__group, __key, __storage_ref.capacity_extent()); - const auto __init_idx = *__probing_iter; - - while (true) - { - const auto __bucket_slots = __storage_ref[*__probing_iter]; - - const auto [__state, __intra_bucket_index] = __find_insert_slot(__key, __bucket_slots); - - if constexpr (!__allows_duplicates) - { - // If the __key is already in the container, return false - if (__group.any(__state == detail::__equal_result::__equal)) - { - return false; - } - } - - const auto __group_contains_available = __group.ballot(__state == detail::__equal_result::__available); - if (__group_contains_available) - { - const auto __src_lane = __ffs(__group_contains_available) - 1; - auto __status = __insert_result::__continue; - if (__group.thread_rank() == __src_lane) - { - __status = - __attempt_insert(__get_slot_ptr(*__probing_iter, __intra_bucket_index), empty_slot_sentinel(), __val); - } - - switch (__group.shfl(__status, __src_lane)) - { - case __insert_result::__success: - return true; - case __insert_result::__duplicate: { - if constexpr (__allows_duplicates) - { - [[fallthrough]]; - } - else - { - return false; - } - } - default: - continue; - } - } - else - { - ++__probing_iter; - if (*__probing_iter == __init_idx) - { - return false; - } - } - } - } - - //! - //! @brief Indicates whether the probe __key `__key` was inserted into the container. - //! - //! @note If the probe __key `__key` was inserted into the container, returns true. Otherwise, returns - //! false. - //! - //! @tparam ProbeKey Probe __key type - //! - //! @param __key The __key to search for - //! - //! @return A boolean indicating whether the probe __key is present - template - [[nodiscard]] _CCCL_DEVICE_API bool contains(_ProbeKey __key) const noexcept - { - static_assert(__cg_size == 1, "Non-CG operation is incompatible with the current probing scheme"); - auto __probing_iter = - __probing_scheme.template make_iterator<__bucket_size>(__key, __storage_ref.capacity_extent()); - const auto __init_idx = *__probing_iter; - - while (true) - { - // TODO atomic_ref::load if insert operator is present - const auto __bucket_slots = __storage_ref[*__probing_iter]; - - for (auto i = 0; i < __bucket_size; ++i) - { - switch (__predicate.template operator()(__key, __extract_key(__bucket_slots[i]))) - { - case detail::__equal_result::__unequal: - continue; - case detail::__equal_result::__empty: - return false; - case detail::__equal_result::__equal: - return true; - } - } - ++__probing_iter; - if (*__probing_iter == __init_idx) - { - return false; - } - } - } - - //! - //! @brief Indicates whether the probe __key `__key` was inserted into the container. - //! - //! @note If the probe __key `__key` was inserted into the container, returns true. Otherwise, returns - //! false. - //! - //! @tparam ProbeKey Probe __key type - //! @tparam ParentCG Type of parent Cooperative Group - //! - //! @param __group The Cooperative Group used to perform group contains - //! @param __key The __key to search for - //! - //! @return A boolean indicating whether the probe __key is present - template - [[nodiscard]] _CCCL_DEVICE_API bool - contains(::cooperative_groups::thread_block_tile<__cg_size, _ParentCG> __group, _ProbeKey __key) const noexcept - { - auto __probing_iter = - __probing_scheme.template make_iterator<__bucket_size>(__group, __key, __storage_ref.capacity_extent()); - const auto __init_idx = *__probing_iter; - - while (true) - { - const auto __bucket_slots = __storage_ref[*__probing_iter]; - - const auto __state = __probe_bucket(__key, __bucket_slots); - - if (__group.any(__state == detail::__equal_result::__equal)) - { - return true; - } - if (__group.any(__state == detail::__equal_result::__empty)) - { - return false; - } - - ++__probing_iter; - if (*__probing_iter == __init_idx) - { - return false; - } - } - } - - //! - //! @brief Finds the slot holding the probe key `__key`. - //! - //! @tparam _ProbeKey Probe key type - //! - //! @param __key The key to search for - //! - //! @return An iterator to the slot holding `__key`, or `end()` if `__key` is not present - template - [[nodiscard]] _CCCL_DEVICE_API __iterator find(_ProbeKey __key) const noexcept - { - static_assert(__cg_size == 1, "Non-CG operation is incompatible with the current probing scheme"); - auto __probing_iter = - __probing_scheme.template make_iterator<__bucket_size>(__key, __storage_ref.capacity_extent()); - const auto __init_idx = *__probing_iter; - - while (true) - { - const auto __bucket_slots = __storage_ref[*__probing_iter]; - - for (::cuda::std::int32_t __i = 0; __i < __bucket_size; ++__i) - { - switch (__predicate.template operator()(__key, __extract_key(__bucket_slots[__i]))) - { - case detail::__equal_result::__empty: - return end(); - case detail::__equal_result::__equal: - return __iterator{__get_slot_ptr(*__probing_iter, __i)}; - default: - continue; - } - } - ++__probing_iter; - if (_CCCL_BUILTIN_EXPECT(*__probing_iter == __init_idx, 0)) - { - return end(); - } - } - } - - //! - //! @brief Cooperative-group variant of `find`. - //! - //! @tparam _ProbeKey Probe key type - //! @tparam _ParentCG Type of parent Cooperative Group - //! - //! @param __group The Cooperative Group used to perform the group find - //! @param __key The key to search for - //! - //! @return An iterator to the slot holding `__key`, or `end()` if `__key` is not present - template - [[nodiscard]] _CCCL_DEVICE_API __iterator - find(::cooperative_groups::thread_block_tile<__cg_size, _ParentCG> __group, _ProbeKey __key) const noexcept - { - auto __probing_iter = - __probing_scheme.template make_iterator<__bucket_size>(__group, __key, __storage_ref.capacity_extent()); - const auto __init_idx = *__probing_iter; - - while (true) - { - const auto __bucket_slots = __storage_ref[*__probing_iter]; - - auto __state = detail::__equal_result::__unequal; - auto __intra_bucket_index = ::cuda::std::int32_t{-1}; - for (::cuda::std::int32_t __i = 0; __i < __bucket_size; ++__i) - { - const auto __res = - __predicate.template operator()(__key, __extract_key(__bucket_slots[__i])); - if (__res != detail::__equal_result::__unequal) - { - __state = __res; - __intra_bucket_index = __i; - break; - } - } - - const auto __group_finds_match = __group.ballot(__state == detail::__equal_result::__equal); - if (__group_finds_match != 0) - { - const auto __src_lane = __ffs(__group_finds_match) - 1; - const auto __probing_index = __group.shfl(*__probing_iter, __src_lane); - const auto __slot_index = __group.shfl(__intra_bucket_index, __src_lane); - return __iterator{__get_slot_ptr(__probing_index, __slot_index)}; - } - if (__group.any(__state == detail::__equal_result::__empty)) - { - return end(); - } - ++__probing_iter; - if (_CCCL_BUILTIN_EXPECT(*__probing_iter == __init_idx, 0)) - { - return end(); - } - } - } - - //! - //! @brief Scans a bucket for the first slot available for inserting @p __key. - //! - //! Returns the intra-bucket index of the first empty slot, or of a slot already holding an equal - //! key when duplicates are disallowed; otherwise reports that the bucket must be skipped. - //! - //! @tparam _ProbeKey Type of the probe key - //! - //! @param __key The key being inserted - //! @param __bucket_slots The bucket to scan - //! - //! @return The probing result for @p __bucket_slots - template - [[nodiscard]] _CCCL_DEVICE_API __bucket_probing_results - __find_insert_slot(const _ProbeKey& __key, __bucket_type __bucket_slots) const noexcept - { - for (::cuda::std::int32_t __i = 0; __i < __bucket_size; ++__i) - { - switch (__predicate.template operator()(__key, __extract_key(__bucket_slots[__i]))) - { - case detail::__equal_result::__available: - return __bucket_probing_results{detail::__equal_result::__available, __i}; - case detail::__equal_result::__equal: - if constexpr (!__allows_duplicates) - { - return __bucket_probing_results{detail::__equal_result::__equal, __i}; - } - break; - default: - break; - } - } - return __bucket_probing_results{detail::__equal_result::__unequal, -1}; - } - - //! - //! @brief Scans a bucket for @p __key during a probe. - //! - //! @tparam _ProbeKey Type of the probe key - //! - //! @param __key The key being queried - //! @param __bucket_slots The bucket to scan - //! - //! @return The first non-`__unequal` result in the bucket, or `__unequal` if every slot differs - //! - template - [[nodiscard]] _CCCL_DEVICE_API detail::__equal_result - __probe_bucket(const _ProbeKey& __key, __bucket_type __bucket_slots) const noexcept - { - auto __res = detail::__equal_result::__unequal; - for (::cuda::std::int32_t __i = 0; __i < __bucket_size; ++__i) - { - __res = __predicate.template operator()(__key, __extract_key(__bucket_slots[__i])); - if (__res != detail::__equal_result::__unequal) - { - return __res; - } - } - return __res; - } - - //! - //! @brief Gets a pointer to the slot at the given probing index and intra-bucket index. - //! - //! @param __probing_idx The current probing index - //! @param __intra_bucket_idx The index within the bucket (0 for flat storage) - //! @return Pointer to the slot - [[nodiscard]] _CCCL_DEVICE_API __value_type* - __get_slot_ptr(__size_type __probing_idx, ::cuda::std::int32_t __intra_bucket_idx) const noexcept - { - return __storage_ref.data() + __probing_idx + __intra_bucket_idx; - } -#endif // _CCCL_CUDA_COMPILATION() - - //! - //! @brief Extracts the __key from a given value type. - //! - //! @tparam Value Input type which is convertible to '__value_type' - //! - //! @param __value The input value - //! - //! @return The __key - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __extract_key(_Value __value) const noexcept - { - if constexpr (__has_payload) - { - return ::thrust::raw_reference_cast(__value).first; - } - else - { - return ::thrust::raw_reference_cast(__value); - } - } - - //! - //! @brief Extracts the payload from a given value type. - //! - //! @note This function is only available if `__has_payload == true` - //! - //! @tparam Value Input type which is convertible to '__value_type' - //! - //! @param __value The input value - //! - //! @return The payload - _CCCL_TEMPLATE(class _Value) - _CCCL_REQUIRES(__has_payload _CCCL_AND(sizeof(_Value) > 0)) - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __extract_payload(_Value __value) const noexcept - { - return ::thrust::raw_reference_cast(__value).second; - } - -#if _CCCL_CUDA_COMPILATION() - //! - //! @brief Converts the given type to the container's native `__value_type`. - //! - //! @tparam T Input type which is convertible to '__value_type' - //! - //! @param __value The input value - //! - //! @return The converted object - template - [[nodiscard]] _CCCL_DEVICE_API constexpr __value_type __native_value(_Value __value) const noexcept - { - if constexpr (__has_payload) - { - return {__key_type{__extract_key(__value)}, __extract_payload(__value)}; - } - else - { - return __value_type{__value}; - } - } - - //! - //! @brief Converts the given type to the container's native `__value_type` while maintaining the - //! heterogeneous __key type. - //! - //! @tparam T Input type which is convertible to '__value_type' - //! - //! @param __value The input value - //! - //! @return The converted object - template - [[nodiscard]] _CCCL_DEVICE_API constexpr auto __heterogeneous_value(_Value __value) const noexcept - { - if constexpr (__has_payload && !::cuda::std::is_same_v<_Value, __value_type>) - { - using mapped_type = decltype(empty_value_sentinel()); - if constexpr (detail::__is_pair_like_v<_Value>) - { - return ::cuda::std::pair{::cuda::std::get<0>(__value), mapped_type{::cuda::std::get<1>(__value)}}; - } - else - { - // hail mary (convert using .first/.second members) - return ::cuda::std::pair{::thrust::raw_reference_cast(__value.first), mapped_type{__value.second}}; - } - } - else - { - return ::thrust::raw_reference_cast(__value); - } - } - - //! - //! @brief Gets the sentinel used to represent an erased slot. - //! - //! @return The sentinel value used to represent an erased slot - [[nodiscard]] _CCCL_DEVICE_API constexpr __value_type __erased_slot_sentinel() const noexcept - { - if constexpr (__has_payload) - { - return ::cuda::std::pair{erased_key_sentinel(), empty_value_sentinel()}; - } - else - { - return erased_key_sentinel(); - } - } - - //! - //! @brief Inserts the specified element with one single CAS operation. - //! - //! @tparam Value Input type which is convertible to '__value_type' - //! - //! @param ____address Pointer to the slot in memory - //! @param ____expected Element to compare against - //! @param ____desired Element to insert - //! - //! @return Result of this operation, i.e., success/continue/duplicate - template - [[nodiscard]] _CCCL_DEVICE_API constexpr __insert_result - packed_cas(__value_type* __address, __value_type __expected, _Value __desired) noexcept - { - using packed_type = - ::cuda::std::__make_nbit_uint_t::digits>; - - auto __slot_ref = ::cuda::atomic_ref{*reinterpret_cast(__address)}; - auto __expected_packed = ::cuda::std::bit_cast(__expected); - const auto __desired_packed = ::cuda::std::bit_cast(__native_value(__desired)); - - const auto success = - __slot_ref.compare_exchange_strong(__expected_packed, __desired_packed, ::cuda::memory_order_relaxed); - - if (success) - { - return __insert_result::__success; - } - else - { - return __predicate.__equal_to(__extract_key(__desired), - __extract_key(::cuda::std::bit_cast<__value_type>(__expected_packed))) - == detail::__equal_result::__equal - ? __insert_result::__duplicate - : __insert_result::__continue; - } - } - - //! - //! @brief Inserts the specified element with two back-to-back CAS operations. - //! - //! @note This CAS is used exclusively to implement the insert operation. - //! - //! @tparam Value Input type which is convertible to '__value_type' - //! - //! @param ____address Pointer to the slot in memory - //! @param ____expected Element to compare against - //! @param ____desired Element to insert - //! - //! @return Result of this operation, i.e., success/continue/duplicate - template - [[nodiscard]] _CCCL_DEVICE_API constexpr __insert_result - back_to_back_cas(__value_type* __address, __value_type __expected, _Value __desired) noexcept - { - using mapped_type = ::cuda::std::decay_t; - - auto __expected_key = __expected.first; - auto __expected_payload = empty_value_sentinel(); - - ::cuda::atomic_ref<__key_type, _Scope> key_ref(__address->first); - ::cuda::atomic_ref payload_ref(__address->second); - - const auto key_cas_success = - key_ref.compare_exchange_strong(__expected_key, __key_type{__desired.first}, ::cuda::memory_order_relaxed); - auto payload_cas_success = - payload_ref.compare_exchange_strong(__expected_payload, __desired.second, ::cuda::memory_order_relaxed); - - // if __key success - if (key_cas_success) - { - while (!payload_cas_success) - { - payload_cas_success = payload_ref.compare_exchange_strong( - __expected_payload = empty_value_sentinel(), __desired.second, ::cuda::memory_order_relaxed); - } - return __insert_result::__success; - } - else if (payload_cas_success) - { - // This is insert-specific, cannot for `erase` operations - payload_ref.store(empty_value_sentinel(), ::cuda::memory_order_relaxed); - } - - // Our __key was already present in the slot, so our __key is a duplicate - // Shouldn't use `predicate` operator directly since it includes a redundant bitwise compare - if (__predicate.__equal_to(__desired.first, __expected_key) == detail::__equal_result::__equal) - { - return __insert_result::__duplicate; - } - - return __insert_result::__continue; - } - - //! - //! @brief Inserts the specified element with CAS-dependent write operations. - //! - //! @tparam Value Input type which is convertible to '__value_type' - //! - //! @param ____address Pointer to the slot in memory - //! @param ____expected Element to compare against - //! @param ____desired Element to insert - //! - //! @return Result of this operation, i.e., success/continue/duplicate - template - [[nodiscard]] _CCCL_DEVICE_API constexpr __insert_result - cas_dependent_write(__value_type* __address, __value_type __expected, _Value __desired) noexcept - { - using mapped_type = ::cuda::std::decay_t; - - ::cuda::atomic_ref<__key_type, _Scope> key_ref(__address->first); - auto __expected_key = __expected.first; - const auto success = - key_ref.compare_exchange_strong(__expected_key, __key_type{__desired.first}, ::cuda::memory_order_relaxed); - - // if __key success - if (success) - { - ::cuda::atomic_ref payload_ref(__address->second); - payload_ref.store(__desired.second, ::cuda::memory_order_relaxed); - return __insert_result::__success; - } - - // Our __key was already present in the slot, so our __key is a duplicate - // Shouldn't use `predicate` operator directly since it includes a redundant bitwise compare - if (__predicate.__equal_to(__desired.first, __expected_key) == detail::__equal_result::__equal) - { - return __insert_result::__duplicate; - } - - return __insert_result::__continue; - } - - //! - //! @brief Attempts to insert an element into a slot. - //! - //! @note Dispatches the correct implementation depending on the container - //! type and presence of other operator mixins. - //! - //! @tparam Value Input type which is convertible to '__value_type' - //! - //! @param ____address Pointer to the slot in memory - //! @param ____expected Element to compare against - //! @param ____desired Element to insert - //! - //! @return Result of this operation, i.e., success/continue/duplicate - template - [[nodiscard]] _CCCL_DEVICE_API __insert_result - __attempt_insert(__value_type* __address, __value_type __expected, _Value __desired) noexcept - { - if constexpr (__has_payload) - { - if constexpr (__has_packable_representation) - { - if (__storage_ref.__is_packed_cas_aligned()) - { - return packed_cas(__address, __expected, __desired); - } - } -# if (__CUDA_ARCH__ < 700) - return cas_dependent_write(__address, __expected, __desired); -# else - return back_to_back_cas(__address, __expected, __desired); -# endif - } - else - { - return packed_cas(__address, __expected, __desired); - } - } - - //! - //! @brief Attempts to insert an element into a slot. - //! - //! @note Dispatches the correct implementation depending on the container - //! type and presence of other operator mixins. - //! - //! @note `stable` indicates that the payload will only be updated once from the sentinel value to the desired value, - //! meaning there can be no ABA situations. - //! - //! @tparam Value Input type which is convertible to '__value_type' - //! - //! @param ____address Pointer to the slot in memory - //! @param ____expected Element to compare against - //! @param ____desired Element to insert - //! - //! @return Result of this operation, i.e., success/continue/duplicate - template - [[nodiscard]] _CCCL_DEVICE_API __insert_result - __attempt_insert_stable(__value_type* __address, __value_type __expected, _Value __desired) noexcept - { - if constexpr (__has_payload) - { - if constexpr (__has_packable_representation) - { - if (__storage_ref.__is_packed_cas_aligned()) - { - return packed_cas(__address, __expected, __desired); - } - } - return cas_dependent_write(__address, __expected, __desired); - } - else - { - return packed_cas(__address, __expected, __desired); - } - } - - //! - //! @brief Waits until the slot payload has been updated - //! - //! @note The function will return once the slot payload is no longer equal to the sentinel - //! Value. - //! - //! @tparam T Map slot type - //! - //! @param __slot The target slot to check payload with - //! @param __sentinel The slot sentinel value - template - _CCCL_DEVICE_API void __wait_for_payload(_Value& __slot, _Value __sentinel) const noexcept - { - auto __ref = ::cuda::atomic_ref<_Value, _Scope>{__slot}; - _Value __current; - // TODO exponential backoff strategy - do - { - __current = __ref.load(::cuda::std::memory_order_relaxed); - } while (detail::__bitwise_compare(__current, __sentinel)); - } -#endif // _CCCL_CUDA_COMPILATION() - - // TODO: Clean up the sentinel handling since it's duplicated in ref and equal wrapper -}; -} // namespace cuda::experimental::cuco::__open_addressing - -#include - -#endif // _CUDAX___CUCO_DETAIL_OPEN_ADDRESSING_REF_IMPL_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/open_addressing/slot_storage_ref.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/open_addressing/slot_storage_ref.cuh deleted file mode 100644 index 47e09440..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/open_addressing/slot_storage_ref.cuh +++ /dev/null @@ -1,126 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_OPEN_ADDRESSING_SLOT_STORAGE_REF_CUH -#define _CUDAX___CUCO_DETAIL_OPEN_ADDRESSING_SLOT_STORAGE_REF_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include - -#include - -namespace cuda::experimental::cuco::__open_addressing -{ -//! @brief Lightweight non-owning reference to a contiguous slot array with bucket abstraction. -//! -//! Provides indexing into the slot array organized as buckets; within each bucket there are -//! `_BucketSize` value-typed slots. The total slot count is carried as a `cuda::std::extents`, so a -//! static `_Capacity` folds the probing reduction to a constant while a dynamic `_Capacity` stores -//! the slot count. The probing layer works in slot offsets bounded by `capacity()`. -//! -//! @tparam _Value The slot value type (e.g. `::cuda::std::pair`) -//! @tparam _BucketSize Number of slots per bucket (compile-time constant) -//! @tparam _Capacity Valid total slot count, or `cuda::std::dynamic_extent` for runtime sizing -template -struct __slot_storage_ref -{ - using __size_type = ::cuda::std::size_t; - using __value_type = _Value; - using __capacity_extent_type = ::cuda::std::extents<__size_type, _Capacity>; - using __iterator = _Value*; - using __const_iterator = const _Value*; - - static constexpr int __bucket_size = _BucketSize; - - using __bucket_type = ::cuda::std::span<_Value, _BucketSize>; - - static_assert(_BucketSize > 0, "bucket size must be greater than zero"); - static_assert(_Capacity == ::cuda::std::dynamic_extent || _Capacity % _BucketSize == 0, - "static capacity must be divisible by the bucket size"); - - _Value* __data_; - _CCCL_NO_UNIQUE_ADDRESS __capacity_extent_type __capacity_; - - //! @brief Constructs a slot storage ref. - //! - //! @param __data Pointer to the first slot - //! @param __capacity Total slot count (must equal the static `_Capacity` when it is static) - _CCCL_HOST_DEVICE_API constexpr __slot_storage_ref(_Value* __data, __size_type __capacity) noexcept - : __data_{__data} - , __capacity_{__capacity} - {} - - //! @brief Returns the bucket at position `__i`. - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __bucket_type operator[](__size_type __i) const noexcept - { - return __bucket_type{__data_ + __i, typename __bucket_type::size_type{_BucketSize}}; - } - - //! @brief Returns the total number of slots. - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __size_type capacity() const noexcept - { - return __capacity_.extent(0); - } - - //! @brief Returns the number of buckets. - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __size_type num_buckets() const noexcept - { - return capacity() / __size_type{_BucketSize}; - } - - //! @brief Returns the total slot count as a `cuda::std::extents` (the probing reduction bound). - //! - //! Returning the extent rather than a plain size keeps the static slot count in the type, so the - //! probing iterator's modular reduction folds to a constant for static `_Capacity`. - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __capacity_extent_type capacity_extent() const noexcept - { - return __capacity_; - } - - //! @brief Returns a pointer to the underlying slot array. - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _Value* data() const noexcept - { - return __data_; - } - - [[nodiscard]] _CCCL_HOST_DEVICE_API bool __is_packed_cas_aligned() const noexcept - { - return ::cuda::is_aligned(__data_, sizeof(_Value)); - } - - //! @brief Returns an iterator to the first slot. - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __iterator begin() const noexcept - { - return __data_; - } - - //! @brief Returns an iterator to one past the last slot. - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __iterator end() const noexcept - { - return __data_ + capacity(); - } -}; -} // namespace cuda::experimental::cuco::__open_addressing - -#include - -#endif // _CUDAX___CUCO_DETAIL_OPEN_ADDRESSING_SLOT_STORAGE_REF_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/prime.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/prime.cuh deleted file mode 100644 index f5a85a59..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/prime.cuh +++ /dev/null @@ -1,173 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_PRIME_CUH -#define _CUDAX___CUCO_DETAIL_PRIME_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include - -#include - -namespace cuda::experimental::cuco::detail -{ -//! @brief Modular multiplication: `(__n1 * __n2) % __m` without overflow. -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint64_t -__mod_mul(::cuda::std::uint64_t __n1, ::cuda::std::uint64_t __n2, ::cuda::std::uint64_t __m) noexcept -{ -#if _CCCL_HAS_INT128() - auto __r = static_cast<__uint128_t>(__n1) * __n2; - return static_cast<::cuda::std::uint64_t>(__r % __m); -#else - // Fallback: Russian-peasant multiplication in modular arithmetic. - ::cuda::std::uint64_t __r = 0; - __n1 %= __m; - __n2 %= __m; - while (__n2 > 0) - { - const ::cuda::std::uint64_t __mod_diff = __m - __n1; - if (__n2 & 1) - { - __r = (__r >= __mod_diff) ? __r - __mod_diff : __r + __n1; - } - __n1 = (__n1 >= __mod_diff) ? __n1 - __mod_diff : __n1 + __n1; - __n2 >>= 1; - } - return __r; -#endif // _CCCL_HAS_INT128() -} - -//! @brief Modular exponentiation: `(__b ^ __e) % __m` via binary exponentiation. -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint64_t -__mod_pow(::cuda::std::uint64_t __b, ::cuda::std::uint64_t __e, ::cuda::std::uint64_t __m) noexcept -{ - ::cuda::std::uint64_t __r = 1; - __b %= __m; - for (; __e > 0; __e >>= 1) - { - if (__e & 1) - { - __r = detail::__mod_mul(__r, __b, __m); - } - __b = detail::__mod_mul(__b, __b, __m); - } - return __r; -} - -//! @brief Single Miller-Rabin witness test. -//! -//! Given `n - 1 == 2^s * d`, checks whether `a^d == 1 (mod n)` or -//! `a^(2^r * d) == n - 1 (mod n)` for some `0 <= r < s`. -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool __miller_rabin_test( - ::cuda::std::uint64_t __n, ::cuda::std::uint64_t __a, ::cuda::std::uint64_t __d, ::cuda::std::uint32_t __s) noexcept -{ - ::cuda::std::uint64_t __x = detail::__mod_pow(__a % __n, __d, __n); - const auto __neg_one = __n - 1; - if (__x == 1 || __x == __neg_one) - { - return true; - } - - for (::cuda::std::uint32_t __i = 1; __i < __s; ++__i) - { - __x = detail::__mod_mul(__x, __x, __n); - if (__x == __neg_one) - { - return true; - } - } - return false; -} - -//! @brief Deterministic primality test for all 64-bit integers. -//! -//! Uses trial division by small primes followed by Miller-Rabin with a fixed -//! set of bases that make the test deterministic for every `uint64_t`. -//! Bases from https://cp-algorithms.com/algebra/primality_tests.html. -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool __is_prime(::cuda::std::uint64_t __n) noexcept -{ - if (__n < 2) - { - return false; - } - - // Trial division by small primes. - constexpr ::cuda::std::uint64_t __small_primes[]{ - 2ull, 3ull, 5ull, 7ull, 11ull, 13ull, 17ull, 19ull, 23ull, 29ull, 31ull, 37ull}; - for (::cuda::std::uint64_t __p : __small_primes) - { - if (__n % __p == 0) - { - return __n == __p; - } - } - - // Decompose `__n - 1 == 2^__s * __d`. - ::cuda::std::uint64_t __d = __n - 1; - ::cuda::std::uint32_t __s = 0; - while ((__d & 1) == 0) - { - __d >>= 1; - ++__s; - } - - // Deterministic witness bases for all `uint64_t` values. - constexpr ::cuda::std::uint64_t __witnesses[]{2ull, 325ull, 9375ull, 28178ull, 450775ull, 9780504ull, 1795265022ull}; - for (::cuda::std::uint64_t __a : __witnesses) - { - if (!detail::__miller_rabin_test(__n, __a, __d, __s)) - { - return false; - } - } - - return true; -} - -//! @brief Returns the smallest prime `>= __n`. -//! -//! For `__n <= 2`, returns 2. Otherwise searches odd numbers starting from -//! `__n` (or `__n + 1` if `__n` is even). -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::uint64_t __next_prime(::cuda::std::uint64_t __n) noexcept -{ - if (__n <= 2) - { - return 2; - } - - __n |= 1; // make odd - - while (!detail::__is_prime(__n)) - { - const auto __next = ::cuda::add_overflow(__n, ::cuda::std::uint64_t{2}); - if (__next.overflow) - { - return __n; - } - __n = __next.value; - } - - return __n; -} -} // namespace cuda::experimental::cuco::detail - -#include - -#endif // _CUDAX___CUCO_DETAIL_PRIME_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/probing_scheme_base.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/probing_scheme_base.cuh deleted file mode 100644 index 14732666..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/probing_scheme_base.cuh +++ /dev/null @@ -1,89 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_PROBING_SCHEME_BASE_CUH -#define _CUDAX___CUCO_DETAIL_PROBING_SCHEME_BASE_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include - -namespace cuda::experimental::cuco::detail -{ -//! @brief Base class of public probing schemes. -//! -//! @tparam _CgSize Cooperative group size -template -class __probing_scheme_base -{ -public: - static constexpr int __cg_size = _CgSize; -}; - -//! @brief Probing iterator class. -//! -//! Yields slot offsets and wraps modulo the total capacity (in slots). The capacity is held as a -//! `cuda::std::extents` so a static slot count folds the reduction to a constant. -//! -//! @tparam _CapacityExtent Capacity extent type (total slots), a `cuda::std::extents` -//! @tparam _StepExtent Probe-step extent type, a `cuda::std::extents` (static for linear probing) -template -class __probing_iterator -{ -public: - using __capacity_extent_type = _CapacityExtent; - using __step_extent_type = _StepExtent; - using __size_type = typename _CapacityExtent::index_type; - - _CCCL_HOST_DEVICE_API constexpr __probing_iterator( - __size_type __start, _StepExtent __step, _CapacityExtent __capacity) noexcept - : __curr_index{__start} - , __step_{__step} - , __capacity_{__capacity} - {} - -#if _CCCL_CUDA_COMPILATION() - _CCCL_DEVICE_API constexpr auto operator*() const noexcept - { - return __curr_index; - } - - _CCCL_DEVICE_API constexpr auto operator++() noexcept - { - __curr_index = (__curr_index + __step_.extent(0)) % __capacity_.extent(0); - return *this; - } - - _CCCL_DEVICE_API constexpr auto operator++(int) noexcept - { - auto __temp = *this; - ++(*this); - return __temp; - } -#endif // _CCCL_CUDA_COMPILATION() - -private: - __size_type __curr_index; - _CCCL_NO_UNIQUE_ADDRESS _StepExtent __step_; - _CCCL_NO_UNIQUE_ADDRESS _CapacityExtent __capacity_; -}; -} // namespace cuda::experimental::cuco::detail - -#include - -#endif // _CUDAX___CUCO_DETAIL_PROBING_SCHEME_BASE_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/utility/cuda.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/utility/cuda.cuh deleted file mode 100644 index 35afd628..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/utility/cuda.cuh +++ /dev/null @@ -1,88 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_UTILITY_CUDA_CUH -#define _CUDAX___CUCO_DETAIL_UTILITY_CUDA_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include - -#include - -#include - -namespace cuda::experimental::cuco::detail -{ -using __index_type = ::cuda::std::int64_t; - -#if _CCCL_CUDA_COMPILATION() -[[nodiscard]] _CCCL_DEVICE_API inline __index_type __global_thread_id() noexcept -{ - return ::cuda::gpu_thread.rank_as<__index_type>(::cuda::grid); -} - -[[nodiscard]] _CCCL_DEVICE_API inline __index_type __grid_stride() noexcept -{ - return ::cuda::gpu_thread.count_as<__index_type>(::cuda::grid); -} -#endif // _CCCL_CUDA_COMPILATION() - -inline constexpr int __default_block_size = 128; - -inline constexpr int __default_stride = 1; - -inline constexpr int __warp_size = 32; - -template -struct __tile_size; - -template <::cuda::std::uint32_t _Size, class _ParentCG> -struct __tile_size<::cooperative_groups::thread_block_tile<_Size, _ParentCG>> -{ - static constexpr int __value = _Size; -}; - -template -inline constexpr int __tile_size_v = __tile_size<_Tile>::__value; - -constexpr _CCCL_HOST_DEVICE_API __index_type __grid_size( - __index_type __num, - int __cg_size = 1, - int __stride = __default_stride, - int __block_size = __default_block_size) noexcept -{ - return ::cuda::ceil_div(__cg_size * __num, __stride * __block_size); -} - -//! @brief Distance helper requiring random access iterators. -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __index_type __distance(_Iterator __begin, _Iterator __end) -{ - static_assert(::cuda::std::random_access_iterator<_Iterator>, "Input iterator should be a random access iterator."); - return __index_type{::cuda::std::distance(__begin, __end)}; -} -} // namespace cuda::experimental::cuco::detail - -#include - -#endif // _CUDAX___CUCO_DETAIL_UTILITY_CUDA_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/utility/strong_type.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/utility/strong_type.cuh deleted file mode 100644 index 2a30ed76..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/utility/strong_type.cuh +++ /dev/null @@ -1,64 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_UTILITY_STRONG_TYPE_CUH -#define _CUDAX___CUCO_DETAIL_UTILITY_STRONG_TYPE_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include - -namespace cuda::experimental::cuco -{ -//! @brief A strong type wrapper. -//! -//! @tparam _Tp Type of the underlying value -template -struct __strong_type -{ - //! @brief Constructs a strong type. - //! - //! @param __v Value to be wrapped as a strong type - _CCCL_HOST_DEVICE_API explicit constexpr __strong_type(_Tp __v) - : __value{__v} - {} - - //! @brief Implicit conversion operator to the underlying value. - //! - //! @return The underlying value - _CCCL_HOST_DEVICE_API constexpr operator _Tp() const noexcept - { - return __value; - } - - _Tp __value; //!< Underlying data value -}; -} // namespace cuda::experimental::cuco - -//! Convenience wrapper for defining a strong type -#define CUDAX_CUCO_DEFINE_STRONG_TYPE(Name, Type) \ - struct Name : __strong_type \ - { \ - _CCCL_HOST_DEVICE_API explicit constexpr Name(Type __value) \ - : __strong_type(__value) \ - {} \ - }; - -#include - -#endif // _CUDAX___CUCO_DETAIL_UTILITY_STRONG_TYPE_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/utility/traits.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/utility/traits.cuh deleted file mode 100644 index bf782696..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/detail/utility/traits.cuh +++ /dev/null @@ -1,45 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_DETAIL_UTILITY_TRAITS_CUH -#define _CUDAX___CUCO_DETAIL_UTILITY_TRAITS_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include - -#include -#include -#include - -#include - -namespace cuda::experimental::cuco::detail -{ -//! @brief Trait value indicating whether `_Tp`, after unwrapping any thrust reference, is a pair-like -//! type (tuple-like with exactly two elements). -//! -//! @tparam _Tp Type to inspect -template -inline constexpr bool __is_pair_like_v = ::cuda::std::__pair_like< - ::cuda::std::remove_reference_t()))>>; -} // namespace cuda::experimental::cuco::detail - -#include - -#endif // _CUDAX___CUCO_DETAIL_UTILITY_TRAITS_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/fixed_capacity_map.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/fixed_capacity_map.cuh deleted file mode 100644 index fc65ced6..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/fixed_capacity_map.cuh +++ /dev/null @@ -1,538 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_FIXED_CAPACITY_MAP_CUH -#define _CUDAX___CUCO_FIXED_CAPACITY_MAP_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -#if !_CCCL_COMPILER(NVRTC) - -namespace cuda::experimental::cuco -{ -//! @brief A GPU-accelerated, unordered, associative container of key-value pairs with unique keys. -//! -//! Allows constant-time inserts and lookups from device code. Many threads may perform -//! the same kind of operation concurrently (e.g. concurrent inserts, or concurrent lookups). -//! Storage is bulk-allocated ahead of time and requires the user to provide sentinel values -//! for empty and, optionally, erased keys. -//! -//! @note Concurrent modification (insert) and lookup (contains) on the same map are not -//! supported: lookups perform non-atomic loads, so a lookup that overlaps a concurrent insert -//! is a data race and results in undefined behavior. Concurrent inserts (with other inserts) -//! and concurrent lookups (with other lookups) are supported; the two kinds must not be mixed. -//! @note `_Capacity` is a span-style `size_t` non-type parameter holding the *valid* (post-rounding) -//! slot count, or `cuda::std::dynamic_extent` (the default) for runtime-sized maps. Obtain a valid -//! value with `cuco::make_valid_capacity`. -//! -//! @tparam _Key Key type. Requires `cuda::is_bitwise_comparable_v<_Key>` -//! @tparam _Tp Mapped value type -//! @tparam _Capacity Requested slot count, or `cuda::std::dynamic_extent` for runtime sizing -//! @tparam _Scope Thread scope for atomic operations -//! @tparam _KeyEqual Key equality comparator -//! @tparam _ProbingScheme Probing scheme type -//! @tparam _BucketSize Slots per bucket -//! @tparam _MemoryResource Memory resource for device storage -template , - class _ProbingScheme = linear_probing<4, hash<_Key>>, - int _BucketSize = 1, - class _MemoryResource = ::cuda::device_memory_pool_ref> -class fixed_capacity_map -{ -public: - using key_type = _Key; ///< Key type - using mapped_type = _Tp; ///< Payload (mapped value) type - using value_type = ::cuda::std::pair<_Key, _Tp>; ///< Key-payload pair type - using size_type = ::cuda::std::size_t; ///< Size type - using key_equal = _KeyEqual; ///< Key equality comparator type - using probing_scheme_type = _ProbingScheme; ///< Probing scheme type - using hasher = typename probing_scheme_type::hasher; ///< Hash function type - - static constexpr auto cg_size = _ProbingScheme::cg_size; ///< Cooperative-group size used for probing - static constexpr auto bucket_size = _BucketSize; ///< Number of slots per bucket - static constexpr auto thread_scope = _Scope; ///< CUDA thread scope for atomic operations - - static_assert(_Capacity == ::cuda::std::dynamic_extent || is_valid_capacity<_ProbingScheme, _BucketSize>(_Capacity), - "Capacity must be a valid open-addressing capacity; obtain it via cuco::make_valid_capacity"); - - //! @brief Valid (post-rounding) slot count; `cuda::std::dynamic_extent` for dynamic maps. - static constexpr size_type capacity_v = _Capacity; - - using ref_type = - fixed_capacity_map_ref<_Key, _Tp, _Scope, _KeyEqual, _ProbingScheme, _BucketSize, _Capacity>; ///< Device - ///< non-owning - ///< ref type - -private: - using __impl_type = __open_addressing:: - __open_addressing_impl<_Key, value_type, _Scope, _KeyEqual, _ProbingScheme, _BucketSize, _MemoryResource>; - - ::cuda::std::unique_ptr<__impl_type> __impl; - mapped_type __empty_value_sentinel; - - //! @brief Synchronizes the CUDA stream. - static void __sync(::cuda::stream_ref __stream) - { - __stream.sync(); - } - -public: - //! @brief Constructs a map with static capacity (encoded in `_Capacity`) and no erasure. - //! - //! @param __stream Stream used for allocation and initialization - //! @param __mr Memory resource for device storage - //! @param __empty_key_sentinel Sentinel indicating an empty key slot - //! @param __empty_value_sentinel Sentinel indicating an empty payload - //! @param __pred Key equality binary callable - //! @param __probing_scheme Probing scheme - _CCCL_TEMPLATE(::cuda::std::size_t _C = _Capacity) - _CCCL_REQUIRES((_C != ::cuda::std::dynamic_extent)) - _CCCL_HOST_API fixed_capacity_map( - ::cuda::stream_ref __stream, - _MemoryResource __mr, - empty_key<_Key> __empty_key_sentinel, - empty_value<_Tp> __empty_value_sentinel, - const _KeyEqual& __pred = {}, - const _ProbingScheme& __probing_scheme = {}) - : __impl{::cuda::std::make_unique<__impl_type>( - __stream, - __mr, - _Capacity, - value_type{key_type(__empty_key_sentinel), mapped_type(__empty_value_sentinel)}, - __pred, - __probing_scheme)} - , __empty_value_sentinel{mapped_type(__empty_value_sentinel)} - {} - - //! @brief Constructs a map with dynamic capacity and no erasure. - //! - //! @param __stream Stream used for allocation and initialization - //! @param __mr Memory resource for device storage - //! @param __capacity Requested slot count (prime/stride-adjusted internally) - //! @param __empty_key_sentinel Sentinel indicating an empty key slot - //! @param __empty_value_sentinel Sentinel indicating an empty payload - //! @param __pred Key equality binary callable - //! @param __probing_scheme Probing scheme - _CCCL_TEMPLATE(::cuda::std::size_t _C = _Capacity) - _CCCL_REQUIRES((_C == ::cuda::std::dynamic_extent)) - _CCCL_HOST_API fixed_capacity_map( - ::cuda::stream_ref __stream, - _MemoryResource __mr, - size_type __capacity, - empty_key<_Key> __empty_key_sentinel, - empty_value<_Tp> __empty_value_sentinel, - const _KeyEqual& __pred = {}, - const _ProbingScheme& __probing_scheme = {}) - : __impl{::cuda::std::make_unique<__impl_type>( - __stream, - __mr, - __capacity, - value_type{key_type(__empty_key_sentinel), mapped_type(__empty_value_sentinel)}, - __pred, - __probing_scheme)} - , __empty_value_sentinel{mapped_type(__empty_value_sentinel)} - {} - - //! @brief Constructs a map sized by a target load factor (dynamic capacity only). - //! - //! @param __stream Stream used for allocation and initialization - //! @param __mr Memory resource for device storage - //! @param __n Expected number of keys - //! @param __desired_load_factor Target load factor in (0, 1] - //! @param __empty_key_sentinel Sentinel indicating an empty key slot - //! @param __empty_value_sentinel Sentinel indicating an empty payload - //! @param __pred Key equality binary callable - //! @param __probing_scheme Probing scheme - _CCCL_TEMPLATE(::cuda::std::size_t _C = _Capacity) - _CCCL_REQUIRES((_C == ::cuda::std::dynamic_extent)) - _CCCL_HOST_API fixed_capacity_map( - ::cuda::stream_ref __stream, - _MemoryResource __mr, - size_type __n, - double __desired_load_factor, - empty_key<_Key> __empty_key_sentinel, - empty_value<_Tp> __empty_value_sentinel, - const _KeyEqual& __pred = {}, - const _ProbingScheme& __probing_scheme = {}) - : __impl{::cuda::std::make_unique<__impl_type>( - __stream, - __mr, - __n, - __desired_load_factor, - value_type{key_type(__empty_key_sentinel), mapped_type(__empty_value_sentinel)}, - __pred, - __probing_scheme)} - , __empty_value_sentinel{mapped_type(__empty_value_sentinel)} - {} - - //! @brief Constructs a map with static capacity and erasure support. - //! - //! @param __stream Stream used for allocation and initialization - //! @param __mr Memory resource for device storage - //! @param __empty_key_sentinel Sentinel indicating an empty key slot - //! @param __empty_value_sentinel Sentinel indicating an empty payload - //! @param __erased_key_sentinel Sentinel indicating an erased key slot - //! @param __pred Key equality binary callable - //! @param __probing_scheme Probing scheme - _CCCL_TEMPLATE(::cuda::std::size_t _C = _Capacity) - _CCCL_REQUIRES((_C != ::cuda::std::dynamic_extent)) - _CCCL_HOST_API fixed_capacity_map( - ::cuda::stream_ref __stream, - _MemoryResource __mr, - empty_key<_Key> __empty_key_sentinel, - empty_value<_Tp> __empty_value_sentinel, - erased_key<_Key> __erased_key_sentinel, - const _KeyEqual& __pred = {}, - const _ProbingScheme& __probing_scheme = {}) - : __impl{::cuda::std::make_unique<__impl_type>( - __stream, - __mr, - _Capacity, - value_type{key_type(__empty_key_sentinel), mapped_type(__empty_value_sentinel)}, - key_type(__erased_key_sentinel), - __pred, - __probing_scheme)} - , __empty_value_sentinel{mapped_type(__empty_value_sentinel)} - {} - - //! @brief Constructs a map with dynamic capacity and erasure support. - //! - //! @param __stream Stream used for allocation and initialization - //! @param __mr Memory resource for device storage - //! @param __capacity Requested slot count (prime/stride-adjusted internally) - //! @param __empty_key_sentinel Sentinel indicating an empty key slot - //! @param __empty_value_sentinel Sentinel indicating an empty payload - //! @param __erased_key_sentinel Sentinel indicating an erased key slot - //! @param __pred Key equality binary callable - //! @param __probing_scheme Probing scheme - _CCCL_TEMPLATE(::cuda::std::size_t _C = _Capacity) - _CCCL_REQUIRES((_C == ::cuda::std::dynamic_extent)) - _CCCL_HOST_API fixed_capacity_map( - ::cuda::stream_ref __stream, - _MemoryResource __mr, - size_type __capacity, - empty_key<_Key> __empty_key_sentinel, - empty_value<_Tp> __empty_value_sentinel, - erased_key<_Key> __erased_key_sentinel, - const _KeyEqual& __pred = {}, - const _ProbingScheme& __probing_scheme = {}) - : __impl{::cuda::std::make_unique<__impl_type>( - __stream, - __mr, - __capacity, - value_type{key_type(__empty_key_sentinel), mapped_type(__empty_value_sentinel)}, - key_type(__erased_key_sentinel), - __pred, - __probing_scheme)} - , __empty_value_sentinel{mapped_type(__empty_value_sentinel)} - {} - - // ===== Clear ===== - - //! @brief Erases all elements from the container. After this call, `size()` returns zero. - //! - //! @param __stream CUDA stream this operation is executed in - void clear(::cuda::stream_ref __stream) - { - __impl->clear(__stream); - } - - //! @brief Asynchronously erases all elements from the container. After this call, `size()` - //! returns zero. - //! - //! @param __stream CUDA stream this operation is executed in - void clear_async(::cuda::stream_ref __stream) noexcept - { - __impl->clear_async(__stream); - } - - // ===== Insert ===== - - //! @brief Inserts all keys in the range `[__first, __last)` and returns the number of successful - //! insertions. - //! - //! @note This function synchronizes the given stream. For asynchronous execution use - //! `insert_async`. - //! - //! @tparam _InputIt Device accessible random access input iterator whose `value_type` is - //! convertible to the map's `value_type` - //! - //! @param __stream CUDA stream used for insert - //! @param __first Beginning of the sequence of keys - //! @param __last End of the sequence of keys - //! - //! @return Number of successful insertions - template - size_type insert(::cuda::stream_ref __stream, _InputIt __first, _InputIt __last) - { - return __impl->insert(__stream, __first, __last, ref()); - } - - //! @brief Asynchronously inserts all keys in the range `[__first, __last)`. - //! - //! @tparam _InputIt Device accessible random access input iterator whose `value_type` is - //! convertible to the map's `value_type` - //! - //! @param __stream CUDA stream used for insert - //! @param __first Beginning of the sequence of keys - //! @param __last End of the sequence of keys - template - void insert_async(::cuda::stream_ref __stream, _InputIt __first, _InputIt __last) noexcept - { - __impl->insert_async(__stream, __first, __last, ref()); - } - - // ===== Contains ===== - - //! @brief Indicates whether each key in `[__first, __last)` is contained in the map. - //! - //! @note This function synchronizes the given stream. For asynchronous execution use - //! `contains_async`. - //! - //! @tparam _InputIt Device accessible input iterator - //! @tparam _OutputIt Device accessible output iterator assignable from `bool` - //! - //! @param __stream CUDA stream used for executing the kernels - //! @param __first Beginning of the sequence of keys - //! @param __last End of the sequence of keys - //! @param __output_begin Beginning of the output sequence of booleans - template - void contains(::cuda::stream_ref __stream, _InputIt __first, _InputIt __last, _OutputIt __output_begin) const - { - contains_async(__stream, __first, __last, __output_begin); - __sync(__stream); - } - - //! @brief Asynchronously indicates whether each key in `[__first, __last)` is contained in the map. - //! - //! @tparam _InputIt Device accessible input iterator - //! @tparam _OutputIt Device accessible output iterator assignable from `bool` - //! - //! @param __stream CUDA stream used for executing the kernels - //! @param __first Beginning of the sequence of keys - //! @param __last End of the sequence of keys - //! @param __output_begin Beginning of the output sequence of booleans - template - void contains_async( - ::cuda::stream_ref __stream, _InputIt __first, _InputIt __last, _OutputIt __output_begin) const noexcept - { - __impl->contains_async(__stream, __first, __last, __output_begin, ref()); - } - - // ===== Find ===== - - //! @brief For each key in `[__first, __last)` writes the associated payload, or `empty_value_sentinel()` - //! if the key is not present. - //! - //! @note This function synchronizes the given stream. For asynchronous execution use `find_async`. - //! - //! @tparam _InputIt Device accessible input iterator - //! @tparam _OutputIt Device accessible output iterator assignable from `mapped_type` - //! - //! @param __stream CUDA stream used for executing the kernels - //! @param __first Beginning of the sequence of keys - //! @param __last End of the sequence of keys - //! @param __output_begin Beginning of the output sequence of payloads - template - void find(::cuda::stream_ref __stream, _InputIt __first, _InputIt __last, _OutputIt __output_begin) const - { - find_async(__stream, __first, __last, __output_begin); - __sync(__stream); - } - - //! @brief Asynchronously, for each key in `[__first, __last)` writes the associated payload, or - //! `empty_value_sentinel()` if the key is not present. - //! - //! @tparam _InputIt Device accessible input iterator - //! @tparam _OutputIt Device accessible output iterator assignable from `mapped_type` - //! - //! @param __stream CUDA stream used for executing the kernels - //! @param __first Beginning of the sequence of keys - //! @param __last End of the sequence of keys - //! @param __output_begin Beginning of the output sequence of payloads - template - void - find_async(::cuda::stream_ref __stream, _InputIt __first, _InputIt __last, _OutputIt __output_begin) const noexcept - { - __impl->find_async(__stream, __first, __last, __output_begin, ref()); - } - - //! @brief For each key `__first[i]` with `__pred(__stencil[i]) == true` writes the associated payload, - //! or `empty_value_sentinel()` if the key is not present; writes `empty_value_sentinel()` for the rest. - //! - //! @note This function synchronizes the given stream. For asynchronous execution use `find_if_async`. - //! - //! @tparam _InputIt Device accessible input iterator - //! @tparam _StencilIt Device accessible random access iterator whose value type is convertible to - //! `_Predicate`'s argument type - //! @tparam _Predicate Unary callable returning `bool` - //! @tparam _OutputIt Device accessible output iterator assignable from `mapped_type` - //! - //! @param __stream CUDA stream used for executing the kernels - //! @param __first Beginning of the sequence of keys - //! @param __last End of the sequence of keys - //! @param __stencil Beginning of the stencil sequence - //! @param __pred Predicate applied to the stencil to determine which keys to query - //! @param __output_begin Beginning of the output sequence of payloads - template - void find_if(::cuda::stream_ref __stream, - _InputIt __first, - _InputIt __last, - _StencilIt __stencil, - _Predicate __pred, - _OutputIt __output_begin) const - { - find_if_async(__stream, __first, __last, __stencil, __pred, __output_begin); - __sync(__stream); - } - - //! @brief Asynchronous version of `find_if`. - //! - //! @tparam _InputIt Device accessible input iterator - //! @tparam _StencilIt Device accessible random access iterator whose value type is convertible to - //! `_Predicate`'s argument type - //! @tparam _Predicate Unary callable returning `bool` - //! @tparam _OutputIt Device accessible output iterator assignable from `mapped_type` - //! - //! @param __stream CUDA stream used for executing the kernels - //! @param __first Beginning of the sequence of keys - //! @param __last End of the sequence of keys - //! @param __stencil Beginning of the stencil sequence - //! @param __pred Predicate applied to the stencil to determine which keys to query - //! @param __output_begin Beginning of the output sequence of payloads - template - void find_if_async( - ::cuda::stream_ref __stream, - _InputIt __first, - _InputIt __last, - _StencilIt __stencil, - _Predicate __pred, - _OutputIt __output_begin) const noexcept - { - __impl->find_if_async(__stream, __first, __last, __stencil, __pred, __output_begin, ref()); - } - - // ===== Accessors ===== - - //! @brief Returns the total number of slots the map can hold (the prime/stride-adjusted capacity). - //! - //! @return Total slot count - [[nodiscard]] constexpr size_type capacity() const noexcept - { - return __impl->capacity(); - } - - //! @brief Gets a device pointer to the underlying slot storage. - //! - //! @return Pointer to the underlying slot storage - [[nodiscard]] _CCCL_HOST_API value_type* data() const - { - return __impl->data(); - } - - //! @brief Gets the sentinel value used to represent an empty key slot. - //! - //! @return The sentinel value used to represent an empty key slot - [[nodiscard]] constexpr key_type empty_key_sentinel() const noexcept - { - return __impl->empty_key_sentinel(); - } - - //! @brief Gets the sentinel value used to represent an empty payload slot. - //! - //! @return The sentinel value used to represent an empty payload slot - [[nodiscard]] constexpr mapped_type empty_value_sentinel() const noexcept - { - return __empty_value_sentinel; - } - - //! @brief Gets the sentinel value used to represent an erased key slot. - //! - //! @return The sentinel value used to represent an erased key slot - [[nodiscard]] constexpr key_type erased_key_sentinel() const noexcept - { - return __impl->erased_key_sentinel(); - } - - //! @brief Gets the function used to compare keys for equality. - //! - //! @return The function used to compare keys for equality - [[nodiscard]] constexpr key_equal key_eq() const noexcept - { - return __impl->key_eq(); - } - - //! @brief Gets the function(s) used to hash keys. - //! - //! @return The function(s) used to hash keys - [[nodiscard]] constexpr hasher hash_function() const noexcept - { - return __impl->hash_function(); - } - - //! @brief Gets a device-usable non-owning reference to this map. - //! - //! The returned ref borrows the map's slot storage and sentinel values and is trivially copyable - //! — safe to pass by value to kernels. The ref's lifetime must not exceed the map's lifetime. - //! - //! @return A `ref_type` referring to this map - [[nodiscard]] auto ref() const noexcept -> ref_type - { - auto __slots = typename ref_type::storage_span_type{__impl->storage_ref().data(), __impl->capacity()}; - return detail::__bitwise_compare(empty_key_sentinel(), erased_key_sentinel()) - ? ref_type{empty_key{empty_key_sentinel()}, - empty_value{empty_value_sentinel()}, - __impl->key_eq(), - __impl->probing_scheme(), - __slots} - : ref_type{empty_key{empty_key_sentinel()}, - empty_value{empty_value_sentinel()}, - erased_key{erased_key_sentinel()}, - __impl->key_eq(), - __impl->probing_scheme(), - __slots}; - } -}; -} // namespace cuda::experimental::cuco - -#endif // !_CCCL_COMPILER(NVRTC) - -#include - -#endif // _CUDAX___CUCO_FIXED_CAPACITY_MAP_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/fixed_capacity_map_ref.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/fixed_capacity_map_ref.cuh deleted file mode 100644 index c150c760..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/fixed_capacity_map_ref.cuh +++ /dev/null @@ -1,354 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_FIXED_CAPACITY_MAP_REF_CUH -#define _CUDAX___CUCO_FIXED_CAPACITY_MAP_REF_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include - -#include - -namespace cuda::experimental::cuco -{ -//! @brief Device non-owning reference type for `fixed_capacity_map`. -//! -//! This lightweight, trivially-copyable reference is intended to be passed by value to device code -//! for performing insert and lookup operations on the hash map. -//! -//! @note Concurrent modify and lookup on the same map are not supported: lookups perform non-atomic -//! loads, so a lookup must not run concurrently with an insert (doing so is a data race). -//! @note cuCollections data structures always place the slot keys on the right-hand side when -//! invoking the key comparison predicate, i.e., `__pred(__query_key, __slot_key)`. -//! @note `_ProbingScheme::cg_size` indicates how many threads are used to handle one independent -//! device operation. `cg_size == 1` uses the scalar (or non-CG) code paths. -//! @note `_Capacity` is a span-style `size_t` non-type parameter encoding the *requested* slot -//! count. Pass `cuda::std::dynamic_extent` (the default) for runtime-sized maps; any concrete -//! value encodes the requested slot count at compile time. The actual slot count is the -//! prime/stride-adjusted value exposed as `capacity_v` and matches the owning map's -//! `fixed_capacity_map::capacity_v` for the same parameters. -//! -//! @tparam _Key Type used for keys -//! @tparam _Tp Type used for mapped values -//! @tparam _Scope The scope in which operations will be performed by individual threads -//! @tparam _KeyEqual Binary callable type used to compare two keys for equality -//! @tparam _ProbingScheme Probing scheme type -//! @tparam _BucketSize Number of slots per bucket -//! @tparam _Capacity Requested slot count, or `cuda::std::dynamic_extent` for runtime sizing -template -class fixed_capacity_map_ref -{ - static_assert(sizeof(_Key) <= 8, "Container does not support key types larger than 8 bytes."); - static_assert(::cuda::is_power_of_two(sizeof(_Key)), "key_type size must be a power of two"); - static_assert(sizeof(_Tp) <= 8, "sizeof(mapped_type) must be no larger than 8 bytes."); - static_assert(::cuda::is_power_of_two(sizeof(::cuda::std::pair<_Key, _Tp>)), - "value_type size must be a power of two"); - static_assert(::cuda::is_bitwise_comparable_v<_Key>, - "Key type must have unique object representations or have been explicitly declared as safe for " - "bitwise comparison via specialization of cuda::is_bitwise_comparable_v."); - - static constexpr bool __allows_duplicates = false; - - static_assert(_Capacity == ::cuda::std::dynamic_extent || is_valid_capacity<_ProbingScheme, _BucketSize>(_Capacity), - "Capacity must be a valid open-addressing capacity; obtain it via cuco::make_valid_capacity"); - -public: - using key_type = _Key; ///< Key type - using mapped_type = _Tp; ///< Payload (mapped value) type - using value_type = ::cuda::std::pair<_Key, _Tp>; ///< Key-payload pair type - using probing_scheme_type = _ProbingScheme; ///< Probing scheme type - using hasher = typename probing_scheme_type::hasher; ///< Hash function type - using size_type = ::cuda::std::size_t; ///< Size type - using key_equal = _KeyEqual; ///< Key equality comparator type - using iterator = value_type*; ///< Slot iterator - using const_iterator = const value_type*; ///< Const slot iterator - - static constexpr auto cg_size = probing_scheme_type::cg_size; ///< Cooperative-group size for probing - static constexpr auto bucket_size = _BucketSize; ///< Number of slots per bucket - static constexpr auto thread_scope = _Scope; ///< CUDA thread scope for atomic operations - - //! @brief Compile-time adjusted slot count; `cuda::std::dynamic_extent` when `_Capacity` is dynamic. - static constexpr size_type capacity_v = _Capacity; - - //! @brief Slot-storage span type. For static `_Capacity`, the span carries the adjusted - //! `capacity_v` extent at compile time; for dynamic `_Capacity`, the extent is dynamic. - using storage_span_type = ::cuda::std::span; - -private: - // Internal adapter to the open-addressing impl. The storage's `_Capacity` template arg receives - // the (already valid) `capacity_v`, so when `_Capacity` is static the slot count travels through - // the storage's extent at compile time and the probing iterator's modular reduction folds to a - // constant. - using __storage_ref_type = __open_addressing::__slot_storage_ref; - - //! @brief Returns the slot count of the given span, validating it for the dynamic case. - //! - //! @param __slots Span over the slot storage - //! - //! @return The total slot count - [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr size_type __checked_capacity(storage_span_type __slots) noexcept - { - if constexpr (_Capacity == ::cuda::std::dynamic_extent) - { - _CCCL_ASSERT((is_valid_capacity<_ProbingScheme, _BucketSize>(__slots.size())), - "storage size is not a valid capacity"); - } - return __slots.size(); - } - - using __impl_type = __open_addressing:: - __open_addressing_ref_impl<_Key, _Scope, _KeyEqual, _ProbingScheme, __storage_ref_type, __allows_duplicates>; - - __impl_type __impl; - -public: - //! @brief Constructs a ref without erasure support. - //! - //! @param __empty_key_sentinel Sentinel indicating an empty key slot - //! @param __empty_value_sentinel Sentinel indicating an empty payload - //! @param __predicate Key equality binary callable - //! @param __probing_scheme Probing scheme - //! @param __slots Span over the slot storage; must contain `capacity()` slots - _CCCL_HOST_DEVICE_API explicit constexpr fixed_capacity_map_ref( - empty_key<_Key> __empty_key_sentinel, - empty_value<_Tp> __empty_value_sentinel, - const _KeyEqual& __predicate, - const _ProbingScheme& __probing_scheme, - storage_span_type __slots) noexcept - : __impl{value_type{key_type(__empty_key_sentinel), mapped_type(__empty_value_sentinel)}, - __predicate, - __probing_scheme, - __storage_ref_type{__slots.data(), __checked_capacity(__slots)}} - {} - - //! @brief Constructs a ref with erasure support. - //! - //! @param __empty_key_sentinel Sentinel indicating an empty key slot - //! @param __empty_value_sentinel Sentinel indicating an empty payload - //! @param __erased_key_sentinel Sentinel indicating an erased key slot - //! @param __predicate Key equality binary callable - //! @param __probing_scheme Probing scheme - //! @param __slots Span over the slot storage; must contain `capacity()` slots - _CCCL_HOST_DEVICE_API explicit constexpr fixed_capacity_map_ref( - empty_key<_Key> __empty_key_sentinel, - empty_value<_Tp> __empty_value_sentinel, - erased_key<_Key> __erased_key_sentinel, - const _KeyEqual& __predicate, - const _ProbingScheme& __probing_scheme, - storage_span_type __slots) noexcept - : __impl{value_type{key_type(__empty_key_sentinel), mapped_type(__empty_value_sentinel)}, - key_type(__erased_key_sentinel), - __predicate, - __probing_scheme, - __storage_ref_type{__slots.data(), __checked_capacity(__slots)}} - {} - - // ===== Accessors ===== - - //! @brief Returns the total number of slots. - //! - //! @return Total slot count (equal to the owning map's `capacity()`) - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr size_type capacity() const noexcept - { - return __impl.capacity(); - } - - //! @brief Returns the sentinel value used to represent an empty key slot. - //! - //! @return The sentinel value used to represent an empty key slot - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr key_type empty_key_sentinel() const noexcept - { - return __impl.empty_key_sentinel(); - } - - //! @brief Returns the sentinel value used to represent an empty payload slot. - //! - //! @return The sentinel value used to represent an empty payload slot - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr mapped_type empty_value_sentinel() const noexcept - { - return __impl.empty_value_sentinel(); - } - - //! @brief Returns the sentinel value used to represent an erased key slot. - //! - //! @return The sentinel value used to represent an erased key slot - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr key_type erased_key_sentinel() const noexcept - { - return __impl.erased_key_sentinel(); - } - - //! @brief Returns the function used to compare keys for equality. - //! - //! @return The key equality comparator - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr key_equal key_eq() const noexcept - { - return __impl.key_eq(); - } - - //! @brief Returns the function(s) used to hash keys. - //! - //! @return The hasher used by this ref's probing scheme - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr hasher hash_function() const noexcept - { - return __impl.hash_function(); - } - - //! @brief Returns the probing scheme used to resolve hash collisions. - //! - //! @return The probing scheme object - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr probing_scheme_type probing_scheme() const noexcept - { - return __impl.probing_scheme(); - } - - //! @brief Returns a const iterator to one past the last slot (the end sentinel). - //! - //! @return Past-the-end const iterator - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const_iterator end() const noexcept - { - return __impl.end(); - } - - //! @brief Returns an iterator to one past the last slot (the end sentinel). - //! - //! @return Past-the-end iterator - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr iterator end() noexcept - { - return __impl.end(); - } - - //! @brief Returns a span over the slot storage backing this ref. - //! - //! @return Span of `capacity()` slots - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr storage_span_type storage_span() const noexcept - { - return storage_span_type{__impl.storage_ref().data(), __impl.capacity()}; - } - -#if _CCCL_CUDA_COMPILATION() - // ===== Insert operations ===== - - //! @brief Inserts a key-value pair. - //! - //! @param __value The key-value pair to insert - //! - //! @return `true` if the pair was inserted, `false` if the key already exists - _CCCL_DEVICE_API bool insert(value_type __value) noexcept - { - return __impl.insert(__value); - } - - //! @brief Inserts a key-value pair using a cooperative group. - //! - //! @tparam _ParentCG Parent cooperative group type - //! - //! @param __group The cooperative group used for this operation - //! @param __value The key-value pair to insert - //! - //! @return `true` if the pair was inserted, `false` if the key already exists - template - _CCCL_DEVICE_API bool - insert(::cooperative_groups::thread_block_tile __group, value_type __value) noexcept - { - return __impl.insert(__group, __value); - } - - // ===== Lookup operations ===== - - //! @brief Checks if a key exists in the map. - //! - //! @param __key The key to search for - //! - //! @return `true` if the key is found - template - [[nodiscard]] _CCCL_DEVICE_API bool contains(_ProbeKey __key) const noexcept - { - return __impl.contains(__key); - } - - //! @brief Cooperative-group variant of `contains`. - //! - //! @tparam _ParentCG Parent cooperative group type - //! @tparam _ProbeKey Probe key type (defaults to `key_type`) - //! - //! @param __group Cooperative group of size `cg_size` performing this lookup - //! @param __key The key to search for - //! - //! @return `true` if the key is found - template - [[nodiscard]] _CCCL_DEVICE_API bool - contains(::cooperative_groups::thread_block_tile __group, _ProbeKey __key) const noexcept - { - return __impl.contains(__group, __key); - } - - //! @brief Finds the slot associated with a key. - //! - //! @tparam _ProbeKey Probe key type (defaults to `key_type`) - //! - //! @param __key The key to search for - //! - //! @return An iterator to the slot holding `__key`, or `end()` if the key is not found - template - [[nodiscard]] _CCCL_DEVICE_API iterator find(_ProbeKey __key) const noexcept - { - return __impl.find(__key); - } - - //! @brief Cooperative-group variant of `find`. - //! - //! @tparam _ParentCG Parent cooperative group type - //! @tparam _ProbeKey Probe key type (defaults to `key_type`) - //! - //! @param __group Cooperative group of size `cg_size` performing this lookup - //! @param __key The key to search for - //! - //! @return An iterator to the slot holding `__key`, or `end()` if the key is not found - template - [[nodiscard]] _CCCL_DEVICE_API iterator - find(::cooperative_groups::thread_block_tile __group, _ProbeKey __key) const noexcept - { - return __impl.find(__group, __key); - } -#endif // _CCCL_CUDA_COMPILATION() -}; -} // namespace cuda::experimental::cuco - -#include - -#endif // _CUDAX___CUCO_FIXED_CAPACITY_MAP_REF_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/hash_functions.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/hash_functions.cuh deleted file mode 100644 index 7d47d129..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/hash_functions.cuh +++ /dev/null @@ -1,97 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_HASH_FUNCTIONS_CUH -#define _CUDAX___CUCO_HASH_FUNCTIONS_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include - -#include - -namespace cuda::experimental::cuco -{ -enum class hash_algorithm -{ - xxhash_32, - xxhash_64, - murmurhash3_32 -#if _CCCL_HAS_INT128() - , - murmurhash3_x86_128, - murmurhash3_x64_128 -#endif // _CCCL_HAS_INT128() -}; - -//! @brief A hash function class specialized for different hash algorithms. -//! -//! @tparam _Key The type of the values to hash -//! @tparam _S The hash strategy to use, defaults to `hash_algorithm::xxhash_32` -template -class hash; - -template -class hash<_Key, hash_algorithm::xxhash_32> : private ::cuda::experimental::cuco::_XXHash_32<_Key> -{ -public: - using ::cuda::experimental::cuco::_XXHash_32<_Key>::_XXHash_32; - using ::cuda::experimental::cuco::_XXHash_32<_Key>::operator(); -}; - -template -class hash<_Key, hash_algorithm::xxhash_64> : private ::cuda::experimental::cuco::_XXHash_64<_Key> -{ -public: - using ::cuda::experimental::cuco::_XXHash_64<_Key>::_XXHash_64; - using ::cuda::experimental::cuco::_XXHash_64<_Key>::operator(); -}; - -template -class hash<_Key, hash_algorithm::murmurhash3_32> : private ::cuda::experimental::cuco::_MurmurHash3_32<_Key> -{ -public: - using ::cuda::experimental::cuco::_MurmurHash3_32<_Key>::_MurmurHash3_32; - using ::cuda::experimental::cuco::_MurmurHash3_32<_Key>::operator(); -}; - -#if _CCCL_HAS_INT128() - -template -class hash<_Key, hash_algorithm::murmurhash3_x86_128> : private ::cuda::experimental::cuco::_MurmurHash3_x86_128<_Key> -{ -public: - using ::cuda::experimental::cuco::_MurmurHash3_x86_128<_Key>::_MurmurHash3_x86_128; - using ::cuda::experimental::cuco::_MurmurHash3_x86_128<_Key>::operator(); -}; - -template -class hash<_Key, hash_algorithm::murmurhash3_x64_128> : private ::cuda::experimental::cuco::_MurmurHash3_x64_128<_Key> -{ -public: - using ::cuda::experimental::cuco::_MurmurHash3_x64_128<_Key>::_MurmurHash3_x64_128; - using ::cuda::experimental::cuco::_MurmurHash3_x64_128<_Key>::operator(); -}; - -#endif // _CCCL_HAS_INT128() -} // namespace cuda::experimental::cuco - -#include - -#endif // _CUDAX___CUCO_HASH_FUNCTIONS_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/hll_policies.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/hll_policies.cuh deleted file mode 100644 index e20c6d52..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/hll_policies.cuh +++ /dev/null @@ -1,26 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_HLL_POLICIES_CUH -#define _CUDAX___CUCO_HLL_POLICIES_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include - -#endif // _CUDAX___CUCO_HLL_POLICIES_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/hyperloglog.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/hyperloglog.cuh deleted file mode 100644 index da6d7177..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/hyperloglog.cuh +++ /dev/null @@ -1,437 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_HYPERLOGLOG_CUH -#define _CUDAX___CUCO_HYPERLOGLOG_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#if !_CCCL_COMPILER(NVRTC) - -namespace cuda::experimental::cuco -{ -//! @brief A GPU-accelerated utility for approximating the number of distinct items in a multiset. -//! -//! @note This implementation is based on the HyperLogLog++ algorithm: -//! https://static.googleusercontent.com/media/research.google.com/de//pubs/archive/40671.pdf. -//! -//! @tparam _Tp Type of items to count -//! @tparam _MemoryResource Type of memory resource used for device storage -//! @tparam _Scope The scope in which operations will be performed by individual threads -//! @tparam _Policy Policy bundling hash function, bit-slicing rule, and finalizer -template > -class hyperloglog -{ -public: - static constexpr auto thread_scope = _Scope; ///< CUDA thread scope - - template <::cuda::thread_scope _NewScope = thread_scope> - using ref_type = hyperloglog_ref<_Tp, _NewScope, _Policy>; ///< Non-owning reference type - - using value_type = typename ref_type<>::value_type; ///< Type of items to count - using policy_type = typename ref_type<>::policy_type; ///< Policy type - using hasher = typename ref_type<>::hasher; ///< Hash function type - using register_type = typename ref_type<>::register_type; ///< HLL register type - - //! A strong type wrapper `sketch_size_kb` of `double`, for specifying the upper-bound - //! sketch size of `cuda::experimental::cuco::hyperloglog(_ref)` in KB. - //! - //! @note Valid sketch sizes are in [0.0625 KB, 1024 KB], which correspond to precision [4, 18]. - using sketch_size_kb = ::cuda::experimental::cuco::__sketch_size_kb_t; - - //! A strong type wrapper `standard_deviation` of `double`, for specifying the desired - //! standard deviation for the cardinality estimate of `cuda::experimental::cuco::hyperloglog(_ref)`. - //! - //! @note Valid standard deviations are approximately in [0.00216, 0.2765], which correspond to - //! precision [4, 18]. - using standard_deviation = ::cuda::experimental::cuco::__standard_deviation_t; - - //! A strong type wrapper `precision` of `int`, for specifying the HyperLogLog precision - //! parameter of `cuda::experimental::cuco::hyperloglog(_ref)`. - //! - //! @note Valid precision values are in [4, 18], which correspond to sketch sizes in - //! [0.0625 KB, 1024 KB] and standard deviations approximately in [0.00216, 0.2765]. - using precision = ::cuda::experimental::cuco::__precision_t; - -private: - ::cuda::device_buffer __sketch_buffer; ///< Storage for sketch - ref_type<> __ref; ///< Device ref of the current `hyperloglog` object - - // Needs to be friends with other instantiations of this class template to have access to their - // storage - template - friend class hyperloglog; - -public: - // TODO enable CTAD - //! @brief Constructs a `hyperloglog` host object. - //! - //! @note Construction is stream-ordered: the initial clear is enqueued on `__stream` without - //! synchronizing it. - //! - //! @param __stream CUDA stream used to initialize the object - //! @param __memory_resource A memory resource used for allocating device storage - //! @param __sketch_size_kb Maximum sketch size in KB - //! @param __policy The policy used to hash items and finalize the estimate - //! - //! @throw If sketch size implies precision outside [4, 18]. - template - _CCCL_HOST_API constexpr hyperloglog( - ::cuda::stream_ref __stream, - _MemoryResource_&& __memory_resource, - sketch_size_kb __sketch_size_kb = sketch_size_kb{32.0}, - const _Policy& __policy = {}) - : hyperloglog{__stream, - ::cuda::std::forward<_MemoryResource_>(__memory_resource), - __to_precision(__sketch_size_kb), - __policy} - {} - - //! @brief Constructs a `hyperloglog` host object. - //! - //! @note Construction is stream-ordered: the initial clear is enqueued on `__stream` without - //! synchronizing it. - //! - //! @param __stream CUDA stream used to initialize the object - //! @param __memory_resource A memory resource used for allocating device storage - //! @param __sd Desired standard deviation for the approximation error - //! @param __policy The policy used to hash items and finalize the estimate - //! - //! @throw If standard deviation implies precision outside [4, 18]. - template - _CCCL_HOST_API constexpr hyperloglog( - ::cuda::stream_ref __stream, - _MemoryResource_&& __memory_resource, - standard_deviation __sd, - const _Policy& __policy = {}) - : hyperloglog{__stream, ::cuda::std::forward<_MemoryResource_>(__memory_resource), __to_precision(__sd), __policy} - {} - - //! @brief Constructs a `hyperloglog` host object. - //! - //! @note Construction is stream-ordered: the initial clear is enqueued on `__stream` without - //! synchronizing it. - //! - //! @param __stream CUDA stream used to initialize the object - //! @param __memory_resource A memory resource used for allocating device storage - //! @param __precision HyperLogLog precision parameter (determines number of registers as 2^precision) - //! @param __policy The policy used to hash items and finalize the estimate - //! - //! @throw If precision is outside [4, 18]. - template - _CCCL_HOST_API constexpr hyperloglog( - ::cuda::stream_ref __stream, - _MemoryResource_&& __memory_resource, - precision __precision, - const _Policy& __policy = {}) - : __sketch_buffer{__stream, - ::cuda::std::forward<_MemoryResource_>(__memory_resource), - ref_type<>::sketch_bytes( - __precision_in_bounds(__precision, "HyperLogLog precision must be in [4, 18]")) - / sizeof(register_type), - ::cuda::no_init} - , __ref{::cuda::std::as_writable_bytes(::cuda::std::span{__sketch_buffer.data(), __sketch_buffer.size()}), - __policy} - { - clear_async(__stream); - } - - _CCCL_HIDE_FROM_ABI ~hyperloglog() = default; - - hyperloglog(const hyperloglog&) = delete; - //! @brief Copy-assignment operator. - //! - //! @return Copy of `*this` - hyperloglog& operator=(const hyperloglog&) = delete; - _CCCL_HIDE_FROM_ABI hyperloglog(hyperloglog&&) = default; ///< Move constructor - - _CCCL_HIDE_FROM_ABI hyperloglog& operator=(hyperloglog&&) = default; - - //! @brief Asynchronously resets the estimator, i.e., clears the current count estimate. - //! - //! @param __stream CUDA stream this operation is executed in - _CCCL_HOST_API constexpr void clear_async(::cuda::stream_ref __stream) noexcept - { - __ref.clear_async(__stream); - } - - //! @brief Resets the estimator, i.e., clears the current count estimate. - //! - //! @note This function synchronizes the given stream. For asynchronous execution use - //! `clear_async`. - //! - //! @param __stream CUDA stream this operation is executed in - _CCCL_HOST_API constexpr void clear(::cuda::stream_ref __stream) - { - __ref.clear(__stream); - } - - //! @brief Asynchronously adds to be counted items to the estimator. - //! - //! @tparam _InputIt Device accessible random access input iterator where - //! std::is_convertible::value_type, - //! _Tp> is `true` - //! - //! @param __stream CUDA stream this operation is executed in - //! @param __first Beginning of the sequence of items - //! @param __last End of the sequence of items - template - _CCCL_HOST_API constexpr void add_async(::cuda::stream_ref __stream, _InputIt __first, _InputIt __last) - { - __ref.add_async(__stream, __first, __last); - } - - //! @brief Adds to be counted items to the estimator. - //! - //! @note This function synchronizes the given stream. For asynchronous execution use - //! `add_async`. - //! - //! @tparam _InputIt Device accessible random access input iterator where - //! std::is_convertible::value_type, - //! _Tp> is `true` - //! - //! @param __stream CUDA stream this operation is executed in - //! @param __first Beginning of the sequence of items - //! @param __last End of the sequence of items - template - _CCCL_HOST_API constexpr void add(::cuda::stream_ref __stream, _InputIt __first, _InputIt __last) - { - __ref.add(__stream, __first, __last); - } - - //! @brief Asynchronously merges the result of `other` estimator into `*this` estimator. - //! - //! @throw If sketch_bytes() != __other.sketch_bytes() - //! - //! @tparam _OtherScope Thread scope of `other` estimator - //! @tparam _OtherMemoryResource Memory resource type of `other` estimator - //! - //! @param __stream CUDA stream this operation is executed in - //! @param __other Other estimator to be merged into `*this` - template <::cuda::thread_scope _OtherScope, class _OtherMemoryResource> - _CCCL_HOST_API constexpr void - merge_async(::cuda::stream_ref __stream, const hyperloglog<_Tp, _OtherMemoryResource, _OtherScope, _Policy>& __other) - { - __ref.merge_async(__stream, __other.__ref); - } - - //! @brief Merges the result of `other` estimator into `*this` estimator. - //! - //! @note This function synchronizes the given stream. For asynchronous execution use - //! `merge_async`. - //! - //! @throw If sketch_bytes() != __other.sketch_bytes() - //! - //! @tparam _OtherScope Thread scope of `other` estimator - //! @tparam _OtherMemoryResource Memory resource type of `other` estimator - //! - //! @param __stream CUDA stream this operation is executed in - //! @param __other Other estimator to be merged into `*this` - template <::cuda::thread_scope _OtherScope, class _OtherMemoryResource> - _CCCL_HOST_API constexpr void - merge(::cuda::stream_ref __stream, const hyperloglog<_Tp, _OtherMemoryResource, _OtherScope, _Policy>& __other) - { - __ref.merge(__stream, __other.__ref); - } - - //! @brief Asynchronously merges the result of `other` estimator reference into `*this` estimator. - //! - //! @throw If sketch_bytes() != __other.sketch_bytes() - //! - //! @tparam _OtherScope Thread scope of `other` estimator - //! - //! @param __stream CUDA stream this operation is executed in - //! @param __other_ref Other estimator reference to be merged into `*this` - template <::cuda::thread_scope _OtherScope> - _CCCL_HOST_API constexpr void merge_async(::cuda::stream_ref __stream, const ref_type<_OtherScope>& __other_ref) - { - __ref.merge_async(__stream, __other_ref); - } - - //! @brief Merges the result of `other` estimator reference into `*this` estimator. - //! - //! @note This function synchronizes the given stream. For asynchronous execution use - //! `merge_async`. - //! - //! @throw If sketch_bytes() != __other.sketch_bytes() - //! - //! @tparam _OtherScope Thread scope of `other` estimator - //! - //! @param __stream CUDA stream this operation is executed in - //! @param __other_ref Other estimator reference to be merged into `*this` - template <::cuda::thread_scope _OtherScope> - _CCCL_HOST_API constexpr void merge(::cuda::stream_ref __stream, const ref_type<_OtherScope>& __other_ref) - { - __ref.merge(__stream, __other_ref); - } - - //! @brief Compute the estimated distinct items count. - //! - //! @note This function synchronizes the given stream. - //! - //! @tparam _MemoryResource Host memory resource used for allocating the host buffer required to - //! compute the final estimate by copying the sketch from device to host - //! - //! @param __stream CUDA stream this operation is executed in - //! @param __host_mr Host memory resource used for copying the sketch - //! - //! @return Approximate distinct items count - template - [[nodiscard]] _CCCL_HOST_API constexpr double - estimate(::cuda::stream_ref __stream, _HostMemoryResource __host_mr = {}) const - { - return __ref.estimate(__stream, __host_mr); - } - - //! @brief Get device ref. - //! - //! @return Device ref object of the current `hyperloglog` host object - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ref_type<> ref() const noexcept - { - return {sketch(), policy()}; - } - - //! @brief Get hash function. - //! - //! @return The hash function - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto hash_function() const noexcept - { - return __ref.hash_function(); - } - - //! @brief Get the policy. - //! - //! @return The policy - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const _Policy& policy() const noexcept - { - return __ref.policy(); - } - - //! @brief Gets the span of the sketch. - //! - //! @return The ::cuda::std::span of the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::span<::cuda::std::byte> sketch() const noexcept - { - return __ref.sketch(); - } - - //! @brief Gets the number of bytes required for the sketch storage. - //! - //! @return The number of bytes required for the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::size_t sketch_bytes() const noexcept - { - return __ref.sketch_bytes(); - } - - //! @brief Gets the number of bytes required for the sketch storage. - //! - //! @param __sketch_size_kb Upper bound sketch size in KB - //! - //! @return The number of bytes required for the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr ::cuda::std::size_t - sketch_bytes(sketch_size_kb __sketch_size_kb) noexcept - { - return ref_type<>::sketch_bytes(__sketch_size_kb); - } - - //! @brief Gets the number of bytes required for the sketch storage. - //! - //! @param __standard_deviation Upper bound standard deviation for approximation error - //! - //! @return The number of bytes required for the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr ::cuda::std::size_t - sketch_bytes(standard_deviation __standard_deviation) noexcept - { - return ref_type<>::sketch_bytes(__standard_deviation); - } - - //! @brief Gets the number of bytes required for the sketch storage. - //! - //! @param __precision HyperLogLog precision parameter - //! - //! @return The number of bytes required for the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr ::cuda::std::size_t sketch_bytes(precision __precision) noexcept - { - return ref_type<>::sketch_bytes(__precision); - } - - //! @brief Gets the alignment required for the sketch storage. - //! - //! @return The required alignment - [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr ::cuda::std::size_t sketch_alignment() noexcept - { - return ref_type<>::sketch_alignment(); - } - -private: - [[nodiscard]] _CCCL_HOST_API static constexpr precision - __precision_in_bounds(precision __precision, const char* __message) - { - const auto __value = static_cast<::cuda::std::int32_t>(__precision); - const auto __in_range = ::cuda::in_range(__value, 4, 18); - if (!__in_range) - { - _CCCL_THROW(::std::invalid_argument, __message); - } - return __precision; - } - - [[nodiscard]] _CCCL_HOST_API static constexpr precision __to_precision(sketch_size_kb __sketch_size_kb) - { - const auto __bytes = ref_type<>::sketch_bytes(__sketch_size_kb) / sizeof(register_type); - const auto __precision = static_cast(::cuda::std::countr_zero(static_cast<::cuda::std::size_t>(__bytes))); - return __precision_in_bounds( - precision{__precision}, "HyperLogLog sketch size must be in range [0.0625 KB, 1024 KB]"); - } - - [[nodiscard]] _CCCL_HOST_API static constexpr precision __to_precision(standard_deviation __standard_deviation) - { - const auto __bytes = ref_type<>::sketch_bytes(__standard_deviation) / sizeof(register_type); - const auto __precision = static_cast(::cuda::std::countr_zero(static_cast<::cuda::std::size_t>(__bytes))); - return __precision_in_bounds( - precision{__precision}, "HyperLogLog standard deviation must be in range [0.00216, 0.2765]"); - } -}; -} // namespace cuda::experimental::cuco - -#endif // !_CCCL_COMPILER(NVRTC) - -#include - -#endif // _CUDAX___CUCO_HYPERLOGLOG_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/hyperloglog_ref.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/hyperloglog_ref.cuh deleted file mode 100644 index 5420c40f..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/hyperloglog_ref.cuh +++ /dev/null @@ -1,364 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_HYPERLOGLOG_REF_CUH -#define _CUDAX___CUCO_HYPERLOGLOG_REF_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include - -namespace cuda::experimental::cuco -{ -//! @brief A non-owning reference to a HyperLogLog sketch for approximating the number of distinct -//! items in a multiset. -//! -//! @note This implementation is based on the HyperLogLog++ algorithm: -//! https://static.googleusercontent.com/media/research.google.com/de//pubs/archive/40671.pdf. -//! -//! @tparam _Tp Type of items to count -//! @tparam _Scope The scope in which operations will be performed by individual threads -//! @tparam _Policy Policy bundling hash function, bit-slicing rule, and finalizer -template > -class hyperloglog_ref -{ - using __impl_type = ::cuda::experimental::cuco::__hyperloglog_impl<_Tp, _Scope, _Policy>; - - __impl_type __impl; ///< Implementation object - - template - friend class hyperloglog_ref; - -public: - static constexpr auto thread_scope = __impl_type::__thread_scope; ///< CUDA thread scope - - using value_type = typename __impl_type::__value_type; ///< Type of items to count - using policy_type = typename __impl_type::__policy_type; ///< Policy type - using hasher = typename __impl_type::__hasher; ///< Type of hash function - using register_type = typename __impl_type::__register_type; ///< HLL register type - - //! A strong type wrapper `sketch_size_kb` of `double`, for specifying the upper-bound - //! sketch size of `cuda::experimental::cuco::hyperloglog(_ref)` in KB. - //! - //! @note Valid sketch sizes are in [0.0625 KB, 1024 KB], which correspond to precision [4, 18]. - using sketch_size_kb = ::cuda::experimental::cuco::__sketch_size_kb_t; - - //! A strong type wrapper `standard_deviation` of `double`, for specifying the desired - //! standard deviation for the cardinality estimate of `cuda::experimental::cuco::hyperloglog(_ref)`. - //! - //! @note Valid standard deviations are approximately in [0.00216, 0.2765], which correspond to - //! precision [4, 18]. - using standard_deviation = ::cuda::experimental::cuco::__standard_deviation_t; - - //! A strong type wrapper `precision` of `int`, for specifying the HyperLogLog precision - //! parameter of `cuda::experimental::cuco::hyperloglog(_ref)`. - //! - //! @note Valid precision values are in [4, 18], which correspond to sketch sizes in - //! [0.0625 KB, 1024 KB] and standard deviations approximately in [0.00216, 0.2765]. - using precision = ::cuda::experimental::cuco::__precision_t; - - template <::cuda::thread_scope _NewScope> - using rebind_scope = hyperloglog_ref<_Tp, _NewScope, _Policy>; ///< Ref type with different thread scope - - //! @brief Constructs a non-owning `hyperloglog_ref` object. - //! - //! @throw If sketch size < 0.0625KB or 64B or standard deviation > 0.2765. Throws if called from - //! host; __trap() if called from device. - //! @throw If sketch size or standard deviation imply precision outside [4, 18]. - //! @throw If sketch storage has insufficient alignment. Throws if called from host; __trap() if called - //! from device. - //! - //! @param __sketch_span Reference to sketch storage - //! @param __policy The policy used to hash items and finalize the estimate - _CCCL_HOST_DEVICE_API constexpr hyperloglog_ref(::cuda::std::span<::cuda::std::byte> __sketch_span, - const _Policy& __policy = {}) - : __impl{__sketch_span, __policy} - {} - - //! @brief Resets the estimator, i.e., clears the current count estimate. - //! - //! @tparam _CG CUDA Cooperative Group type - //! - //! @param __group CUDA Cooperative group this operation is executed in - _CCCL_TEMPLATE(class _CG) - _CCCL_REQUIRES((!::cuda::std::is_convertible_v<_CG, ::cuda::stream_ref>) ) - _CCCL_DEVICE_API constexpr void clear(_CG __group) noexcept - { - // The constraint above is to work around an incompatibility between host and device - // overload preference for clang and NVCC. See - // https://llvm.org/docs/CompileCudaWithLLVM.html#overloading-based-on-host-and-device-attributes - // for further reading, but the bottom line is when: - // - // 1. Compiling in device mode (and clang compiles CUDA in a "hybrid" host-device mode, - // also explained by the link above). - // 2. And the current function is __host__ __device__. - // 3. And the function whose overload needs to be resolved has both a __host__ __device__, - // and __device__ (and/or __host__) overload. - // - // Then clang will prefer these overloads (assuming they have equal priority under C++ - // rules) in the following order: - // - // 1. __host__ __device__ - // 2. __device__ - // 3. __host__ - // - // In this particular case, `clear(_CG)` conflicts with `clear(::cuda::stream_ref)` when called - // from `hyperloglog::clear(::cuda::stream_ref)`. `hyperloglog::clear(::cuda::stream_ref)` - // is constexpr, and therefore implicitly __host__ __device__. Since - // `clear(::cuda::stream_ref)` on this class is only __host__, it will take lower priority - // that `clear(_CG)`, and we get: - // - // cudax/include/cuda/experimental/__cuco/detail/hyperloglog/hyperloglog_impl.cuh:131:28: error: no member named - // 'thread_rank' in 'cuda::stream_ref' [clang-diagnostic-error] - // - // 131 | for (int __i = __group.thread_rank(); __i < __sketch.size(); __i += __group.size()) - // | ~~~~~~~ ^ - __impl.__clear(__group); - } - - //! @brief Asynchronously resets the estimator, i.e., clears the current count estimate. - //! - //! @param __stream CUDA stream this operation is executed in - _CCCL_HOST_API constexpr void clear_async(::cuda::stream_ref __stream) noexcept - { - __impl.__clear_async(__stream); - } - - //! @brief Resets the estimator, i.e., clears the current count estimate. - //! - //! @note This function synchronizes the given stream. For asynchronous execution use - //! `clear_async`. - //! - //! @param __stream CUDA stream this operation is executed in - _CCCL_HOST_API constexpr void clear(::cuda::stream_ref __stream) - { - __impl.__clear(__stream); - } - - //! @brief Adds an item to the estimator. - //! - //! @param __item The item to be counted - _CCCL_DEVICE_API constexpr void add(const _Tp& __item) noexcept - { - __impl.__add(__item); - } - - //! @brief Asynchronously adds to be counted items to the estimator. - //! - //! @tparam _InputIt Device accessible random access input iterator where - //! std::is_convertible::value_type, - //! _Tp> is `true` - //! - //! @param __stream CUDA stream this operation is executed in - //! @param __first Beginning of the sequence of items - //! @param __last End of the sequence of items - template - _CCCL_HOST_API constexpr void add_async(::cuda::stream_ref __stream, _InputIt __first, _InputIt __last) - { - __impl.__add_async(__first, __last, __stream); - } - - //! @brief Adds to be counted items to the estimator. - //! - //! @note This function synchronizes the given stream. For asynchronous execution use - //! `add_async`. - //! - //! @tparam _InputIt Device accessible random access input iterator where - //! std::is_convertible::value_type, - //! _Tp> is `true` - //! - //! @param __stream CUDA stream this operation is executed in - //! @param __first Beginning of the sequence of items - //! @param __last End of the sequence of items - template - _CCCL_HOST_API constexpr void add(::cuda::stream_ref __stream, _InputIt __first, _InputIt __last) - { - __impl.__add(__first, __last, __stream); - } - - //! @brief Merges the result of `other` estimator reference into `*this` estimator reference. - //! - //! @throw If sketch_bytes() != __other.sketch_bytes(), then terminates execution with a device __trap() - //! - //! @tparam _CG CUDA Cooperative Group type - //! @tparam _OtherScope Thread scope of `other` estimator - //! - //! @param __group CUDA Cooperative group this operation is executed in - //! @param __other Other estimator reference to be merged into `*this` - _CCCL_TEMPLATE(class _CG, ::cuda::thread_scope _OtherScope) - _CCCL_REQUIRES((!::cuda::std::is_convertible_v<_CG, ::cuda::stream_ref>) ) - _CCCL_DEVICE_API constexpr void merge(_CG __group, const hyperloglog_ref<_Tp, _OtherScope, _Policy>& __other) - { - // The constraint above works around the same host/device overload preference issue as - // documented in `clear(_CG)`: `merge(_CG, ...)` would otherwise conflict with - // `merge(::cuda::stream_ref, ...)` when called from `hyperloglog::merge(::cuda::stream_ref, ...)`. - __impl.__merge(__group, __other.__impl); - } - - //! @brief Asynchronously merges the result of `other` estimator reference into `*this` - //! estimator. - //! - //! @throw If sketch_bytes() != __other.sketch_bytes() - //! - //! @tparam _OtherScope Thread scope of `other` estimator - //! - //! @param __stream CUDA stream this operation is executed in - //! @param __other Other estimator reference to be merged into `*this` - template <::cuda::thread_scope _OtherScope> - _CCCL_HOST_API constexpr void - merge_async(::cuda::stream_ref __stream, const hyperloglog_ref<_Tp, _OtherScope, _Policy>& __other) - { - __impl.__merge_async(__other.__impl, __stream); - } - - //! @brief Merges the result of `other` estimator reference into `*this` estimator. - //! - //! @note This function synchronizes the given stream. For asynchronous execution use - //! `merge_async`. - //! - //! @throw If sketch_bytes() != __other.sketch_bytes() - //! - //! @tparam _OtherScope Thread scope of `other` estimator - //! - //! @param __stream CUDA stream this operation is executed in - //! @param __other Other estimator reference to be merged into `*this` - template <::cuda::thread_scope _OtherScope> - _CCCL_HOST_API constexpr void - merge(::cuda::stream_ref __stream, const hyperloglog_ref<_Tp, _OtherScope, _Policy>& __other) - { - __impl.__merge(__other.__impl, __stream); - } - - //! @brief Compute the estimated distinct items count. - //! - //! @param __group CUDA thread block group this operation is executed in - //! - //! @return Approximate distinct items count - [[nodiscard]] _CCCL_DEVICE_API double estimate(const ::cooperative_groups::thread_block& __group) const noexcept - { - return __impl.__estimate(__group); - } - - //! @brief Compute the estimated distinct items count. - //! - //! @note This function synchronizes the given stream. - //! - //! @tparam _HostMemoryResource Host memory resource used for allocating the host buffer required to - //! compute the final estimate by copying the sketch from device to host - //! - //! @param __stream CUDA stream this operation is executed in - //! @param __host_mr Host memory resource used for copying the sketch - //! - //! @return Approximate distinct items count - template - [[nodiscard]] _CCCL_HOST_API constexpr double - estimate(::cuda::stream_ref __stream, _HostMemoryResource __host_mr = {}) const - { - return __impl.__estimate(__host_mr, __stream); - } - - //! @brief Gets the hash function. - //! - //! @return The hash function - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto hash_function() const noexcept - { - return __impl.__hash_function(); - } - - //! @brief Gets the policy. - //! - //! @return The policy - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const _Policy& policy() const noexcept - { - return __impl.__policy_(); - } - - //! @brief Gets the span of the sketch. - //! - //! @return The ::cuda::std::span of the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::span<::cuda::std::byte> sketch() const noexcept - { - return __impl.__sketch_span(); - } - - //! @brief Gets the number of bytes required for the sketch storage. - //! - //! @return The number of bytes required for the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::std::size_t sketch_bytes() const noexcept - { - return __impl.__sketch_bytes(); - } - - //! @brief Gets the number of bytes required for the sketch storage. - //! - //! @param __sketch_size_kb Upper bound sketch size in KB - //! - //! @return The number of bytes required for the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr ::cuda::std::size_t - sketch_bytes(sketch_size_kb __sketch_size_kb) noexcept - { - return __impl_type::__sketch_bytes(__sketch_size_kb); - } - - //! @brief Gets the number of bytes required for the sketch storage. - //! - //! @param __standard_deviation Upper bound standard deviation for approximation error - //! - //! @return The number of bytes required for the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr ::cuda::std::size_t - sketch_bytes(standard_deviation __standard_deviation) noexcept - { - return __impl_type::sketch_bytes(__standard_deviation); - } - - //! @brief Gets the number of bytes required for the sketch storage. - //! - //! @param __precision HyperLogLog precision parameter - //! - //! @return The number of bytes required for the sketch - [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr ::cuda::std::size_t sketch_bytes(precision __precision) noexcept - { - return __impl_type::sketch_bytes(__precision); - } - - //! @brief Gets the alignment required for the sketch storage. - //! - //! @return The required alignment - [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr ::cuda::std::size_t sketch_alignment() noexcept - { - return __impl_type::__sketch_alignment(); - } -}; -} // namespace cuda::experimental::cuco - -#include - -#endif // _CUDAX___CUCO_HYPERLOGLOG_REF_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/probing_scheme.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/probing_scheme.cuh deleted file mode 100644 index 851df046..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/probing_scheme.cuh +++ /dev/null @@ -1,273 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_PROBING_SCHEME_CUH -#define _CUDAX___CUCO_PROBING_SCHEME_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -namespace cuda::experimental::cuco -{ -//! @brief Public linear probing scheme class. -//! -//! @note Linear probing is efficient when few collisions are present, e.g., low occupancy or low -//! multiplicity. -//! -//! @note `_Hash` should be a callable object type. -//! -//! @tparam _CgSize Cooperative group size -//! @tparam _Hash Hash functor type -template -class linear_probing : detail::__probing_scheme_base<_CgSize> -{ - using __base_type = detail::__probing_scheme_base<_CgSize>; - -public: - static constexpr int cg_size = __base_type::__cg_size; - using hasher = _Hash; - - //! @brief Constructs a linear probing scheme with the given hasher callable. - //! - //! @param __hash Hasher - _CCCL_HOST_DEVICE_API constexpr linear_probing(const _Hash& __hash = {}) - : __hash{__hash} - {} - - //! @brief Makes a copy of the current probing scheme with the given hasher. - //! - //! @tparam _NewHash New hasher type - //! - //! @param __hash Hasher - //! - //! @return Copy of the current probing scheme - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto rebind_hash_function(const _NewHash& __hash) const noexcept - { - return linear_probing{__hash}; - } - - //! @brief Returns a probing iterator. - //! - //! @tparam _BucketSize Size of the bucket - //! @tparam _ProbeKey Type of probing key - //! @tparam _Capacity Capacity extent type (total slots) - //! - //! @param __probe_key The probing key - //! @param __cap Capacity extent (total slots) bounding the iteration - //! - //! @return An iterator whose value_type is convertible to the slot index type - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto make_iterator(_ProbeKey __probe_key, _Capacity __cap) const noexcept - { - using __size_type = typename _Capacity::index_type; - using __step_extent = ::cuda::std::extents<__size_type, _BucketSize>; - const __size_type __init = __hash(__probe_key) % (__cap.extent(0) / _BucketSize) * _BucketSize; - return detail::__probing_iterator<_Capacity, __step_extent>{__init, __step_extent{}, __cap}; - } - - //! @brief Returns a cooperative group based probing iterator. - //! - //! @tparam _BucketSize Size of the bucket - //! @tparam _ProbeKey Type of probing key - //! @tparam _Capacity Capacity extent type (total slots) - //! @tparam _ParentCG Type of parent cooperative group - //! - //! @param __group The cooperative group used to generate the probing iterator - //! @param __probe_key The probing key - //! @param __cap Capacity extent (total slots) bounding the iteration - //! - //! @return An iterator whose value_type is convertible to the slot index type - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto - make_iterator(::cooperative_groups::thread_block_tile __group, - _ProbeKey __probe_key, - _Capacity __cap) const noexcept - { - using __size_type = typename _Capacity::index_type; - constexpr __size_type __stride = cg_size * _BucketSize; - using __step_extent = ::cuda::std::extents<__size_type, __stride>; - const __size_type __init = - __hash(__probe_key) % (__cap.extent(0) / __stride) * __stride + __size_type{__group.thread_rank() * _BucketSize}; - return detail::__probing_iterator<_Capacity, __step_extent>{__init, __step_extent{}, __cap}; - } - - //! @brief Gets the function used to hash keys. - //! - //! @return The function used to hash keys - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr hasher hash_function() const noexcept - { - return __hash; - } - -private: - _Hash __hash; -}; - -//! @brief Public double hashing scheme class. -//! -//! @note Default probing scheme for cuco data structures. It shows superior performance over linear -//! probing especially when dealing with high multiplicity and/or high occupancy use cases. -//! -//! @note `_Hash1` and `_Hash2` should be callable object types. -//! -//! @note `_Hash2` needs to be able to construct from an integer value to avoid secondary clustering. -//! -//! @tparam _CgSize Cooperative group size -//! @tparam _Hash1 First hash functor -//! @tparam _Hash2 Second hash functor -template -class double_hashing : detail::__probing_scheme_base<_CgSize> -{ - using __base_type = detail::__probing_scheme_base<_CgSize>; - -public: - static constexpr int cg_size = __base_type::__cg_size; - using hasher = ::cuda::std::tuple<_Hash1, _Hash2>; - - //! @brief Constructs a double hashing probing scheme with the two hasher callables. - //! - //! @param __hash1 First hasher - //! @param __hash2 Second hasher - _CCCL_HOST_DEVICE_API constexpr double_hashing(const _Hash1& __hash1 = {}, const _Hash2& __hash2 = {1}) - : __hash1{__hash1} - , __hash2{__hash2} - {} - - //! @brief Constructs a double hashing probing scheme with the given hasher tuple. - //! - //! @param __hash Hasher tuple - _CCCL_HOST_DEVICE_API constexpr double_hashing(const ::cuda::std::tuple<_Hash1, _Hash2>& __hash) - : __hash1{::cuda::std::get<0>(__hash)} - , __hash2{::cuda::std::get<1>(__hash)} - {} - - //! @brief Makes a copy of the current probing scheme with the given hasher. - //! - //! @tparam _NewHash Tuple-like new hasher type - //! - //! @param __hash Hasher - //! - //! @return Copy of the current probing scheme - _CCCL_TEMPLATE(class _NewHash) - _CCCL_REQUIRES(::cuda::std::__tuple_like<_NewHash>) - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto rebind_hash_function(const _NewHash& __hash) const - { - static_assert(::cuda::std::__tuple_like<_NewHash> && ::cuda::std::tuple_size<_NewHash>::value == 2, - "The given hasher must be a tuple-like object with exactly two elements"); - - const auto& [__hash1, __hash2] = __hash; - using __hash1_type = ::cuda::std::decay_t; - using __hash2_type = ::cuda::std::decay_t; - return double_hashing{__hash1, __hash2}; - } - - //! @brief Returns a probing iterator. - //! - //! @tparam _BucketSize Size of the bucket - //! @tparam _ProbeKey Type of probing key - //! @tparam _Capacity Capacity extent type (total slots) - //! - //! @param __probe_key The probing key - //! @param __cap Capacity extent (total slots) bounding the iteration - //! - //! @return An iterator whose value_type is convertible to the slot index type - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto make_iterator(_ProbeKey __probe_key, _Capacity __cap) const noexcept - { - using __size_type = typename _Capacity::index_type; - using __step_extent = ::cuda::std::extents<__size_type, ::cuda::std::dynamic_extent>; - return detail::__probing_iterator<_Capacity, __step_extent>{ - __size_type{__hash1(__probe_key)} % (__cap.extent(0) / _BucketSize) * _BucketSize, - __step_extent{__size_type{(__hash2(__probe_key) % (__cap.extent(0) / _BucketSize - 1) + 1) * _BucketSize}}, - __cap}; - } - - //! @brief Returns a cooperative group based probing iterator. - //! - //! @tparam _BucketSize Size of the bucket - //! @tparam _ProbeKey Type of probing key - //! @tparam _Capacity Capacity extent type (total slots) - //! @tparam _ParentCG Type of parent cooperative group - //! - //! @param __group The cooperative group used to generate the probing iterator - //! @param __probe_key The probing key - //! @param __cap Capacity extent (total slots) bounding the iteration - //! - //! @return An iterator whose value_type is convertible to the slot index type - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto - make_iterator(::cooperative_groups::thread_block_tile __group, - _ProbeKey __probe_key, - _Capacity __cap) const noexcept - { - using __size_type = typename _Capacity::index_type; - constexpr __size_type __stride = cg_size * _BucketSize; - using __step_extent = ::cuda::std::extents<__size_type, ::cuda::std::dynamic_extent>; - - return detail::__probing_iterator<_Capacity, __step_extent>{ - __size_type{__hash1(__probe_key)} % (__cap.extent(0) / __stride) * __stride - + __size_type{__group.thread_rank() * _BucketSize}, - __step_extent{__size_type{(__hash2(__probe_key) % (__cap.extent(0) / __stride - 1) + 1) * __stride}}, - __cap}; - } - - //! @brief Gets the functions used to hash keys. - //! - //! @return The functions used to hash keys - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr hasher hash_function() const noexcept - { - return {__hash1, __hash2}; - } - -private: - _Hash1 __hash1; - _Hash2 __hash2; -}; - -//! @brief Trait value indicating whether a probing scheme is double hashing. -//! -//! @tparam _Tp Input probing scheme type -template -inline constexpr bool is_double_hashing_v = false; - -//! @brief Specialization indicating that `double_hashing` is a double hashing scheme. -//! -//! @tparam _CgSize Cooperative group size -//! @tparam _Hash1 First hash functor -//! @tparam _Hash2 Second hash functor -template -inline constexpr bool is_double_hashing_v> = true; -} // namespace cuda::experimental::cuco - -#include - -#endif // _CUDAX___CUCO_PROBING_SCHEME_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cuco/types.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cuco/types.cuh deleted file mode 100644 index 042e2c57..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cuco/types.cuh +++ /dev/null @@ -1,66 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDAX___CUCO_TYPES_CUH -#define _CUDAX___CUCO_TYPES_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include - -#include - -namespace cuda::experimental::cuco -{ -//! @brief Strong type wrapper for an empty key sentinel. -//! -//! @tparam _Key The key type -template -struct empty_key : __strong_type<_Key> -{ - _CCCL_HOST_DEVICE_API explicit constexpr empty_key(_Key __value) noexcept - : __strong_type<_Key>(__value) - {} -}; - -//! @brief Strong type wrapper for an empty value sentinel. -//! -//! @tparam _Tp The mapped value type -template -struct empty_value : __strong_type<_Tp> -{ - _CCCL_HOST_DEVICE_API explicit constexpr empty_value(_Tp __value) noexcept - : __strong_type<_Tp>(__value) - {} -}; - -//! @brief Strong type wrapper for an erased key sentinel. -//! -//! @tparam _Key The key type -template -struct erased_key : __strong_type<_Key> -{ - _CCCL_HOST_DEVICE_API explicit constexpr erased_key(_Key __value) noexcept - : __strong_type<_Key>(__value) - {} -}; -} // namespace cuda::experimental::cuco - -#include - -#endif // _CUDAX___CUCO_TYPES_CUH diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cufile/cufile.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cufile/cufile.cuh deleted file mode 100644 index cf694c9c..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cufile/cufile.cuh +++ /dev/null @@ -1,311 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include - -#include -#include -#include -#include - -namespace cuda::experimental -{ -//! @brief An owning wrapper of \c CUfileHandle_t and the OS specific native file handle. -class cufile : public cufile_ref -{ -public: - using native_handle_type = __cufile_os_native_type; //!< The underlying OS native handle type. - -private: - using __oflags_type = int; - - static constexpr native_handle_type __invalid_native_handle = -1; - - native_handle_type __native_handle_{__invalid_native_handle}; //< The native handle. - - //! @brief Constructs the object from native handle and cuFile file handle. - _CCCL_HIDE_FROM_ABI cufile(cufile_ref __cufile_handle, native_handle_type __native_handle) noexcept - : cufile_ref{__cufile_handle} - , __native_handle_{__native_handle} - {} - - //! @brief Make open flags from the \c cuda::cufile_open_mode. - //! - //! @param __om The cuFile open mode. - //! - //! @return The flags mask to be passed to open function. - [[nodiscard]] static _CCCL_HOST_API constexpr __oflags_type __make_oflags(cufile_open_mode __om) noexcept - { - __oflags_type __ret{}; - if ((__om & (cufile_open_mode::in | cufile_open_mode::out)) == (cufile_open_mode::in | cufile_open_mode::out)) - { - __ret |= O_RDWR | O_CREAT; - } - else if ((__om & cufile_open_mode::in) == cufile_open_mode::in) - { - __ret |= O_RDONLY; - } - else if ((__om & cufile_open_mode::out) == cufile_open_mode::out) - { - __ret |= O_WRONLY | O_CREAT; - } - - __ret |= ((__om & cufile_open_mode::trunc) == cufile_open_mode::trunc) ? O_TRUNC : 0; - __ret |= ((__om & cufile_open_mode::noreplace) == cufile_open_mode::noreplace) ? O_EXCL : 0; - __ret |= ((__om & cufile_open_mode::direct) == cufile_open_mode::direct) ? O_DIRECT : 0; - return __ret; - } - - //! @brief Wrapper for opening the native handle. - [[nodiscard]] static _CCCL_HOST_API native_handle_type __open_file(const char* __filename, __oflags_type __oflags) - { - // if O_CREAT flag is specified, use the same mode as if opened by fopend - ::mode_t __ocreat_mode{}; - if (__oflags & O_CREAT) - { - __ocreat_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; - } - - int __fd = ::open(__filename, __oflags, __ocreat_mode); - - if (__fd == -1) - { - errno = 0; // clear errno - _CCCL_THROW(::std::runtime_error, "Failed to open file."); - } - - return __fd; - } - - //! @brief Wrapper for retrieving the open mode. - [[nodiscard]] static _CCCL_HOST_API cufile_open_mode __open_mode(native_handle_type __native_handle) - { - int __oflags = ::fcntl(__native_handle, F_GETFL); - - if (__oflags == -1) - { - errno = 0; // clear errno - _CCCL_THROW(::std::runtime_error, "Failed to retrieve open flags."); - } - - cufile_open_mode __om{}; - if (__oflags & O_RDWR) - { - __om |= cufile_open_mode::in | cufile_open_mode::out; - } - else if (__oflags & O_RDONLY) - { - __om |= cufile_open_mode::in; - } - else if (__oflags & O_WRONLY) - { - __om |= cufile_open_mode::out; - } - __om |= (__oflags & O_TRUNC) ? cufile_open_mode::trunc : cufile_open_mode{}; - __om |= (__oflags & O_EXCL) ? cufile_open_mode::noreplace : cufile_open_mode{}; - __om |= (__oflags & O_DIRECT) ? cufile_open_mode::direct : cufile_open_mode{}; - return __om; - } - - //! @brief Wrapper for closing the native handle. - [[nodiscard]] static _CCCL_HOST_API bool __close_file_no_throw(native_handle_type __native_handle) noexcept - { - return ::close(__native_handle) == 0; - } - - //! @brief Wrapper for closing the native handle. Throws \c cuda::std::runtime_error if an error occurs. - static _CCCL_HOST_API void __close_file(native_handle_type __native_handle) - { - if (!__close_file_no_throw(__native_handle)) - { - errno = 0; // clear errno - _CCCL_THROW(::std::runtime_error, "Failed to close file."); - } - } - -public: - //! @brief Make a cufile object from already existing native handle. - //! - // The ownership of the handle is transferred to the object and the handle is registered by the cuFile driver. - //! - //! @param __native_handle The native handle. - //! - //! @return The created cufile object. - [[nodiscard]] static _CCCL_HOST_API cufile from_native_handle(native_handle_type __native_handle) - { - return cufile{cufile_driver.register_native_handle(__native_handle), __native_handle}; - } - - _CCCL_HIDE_FROM_ABI cufile() noexcept = default; - - //! @brief Constructs the object by opening file @c __filename in mode @c __open_mode. - //! - //! @param __filename Path to the file. Must be a zero terminated string. - //! @param __open_mode Open mode to open the file with. - //! - //! @throws cuda::std::runtime_error if the file cannot be opened. - //! @throws cuda::cuda_error if a CUDA driver error occurs. - //! @throws cuda::cufile_error if a cuFile driver error occurs. - _CCCL_HOST_API cufile(const char* __filename, cufile_open_mode __open_mode) - { - __native_handle_ = __open_file(__filename, __make_oflags(__open_mode)); - try - { - __cufile_handle_ = cufile_driver.register_native_handle(__native_handle_).get(); - } - catch (...) - { - __close_file(__native_handle_); - throw; - } - } - - cufile(const cufile&) = delete; - - //! @brief Move-construct a new @c cufile. - //! - //! @param __other The other @c cufile. - //! - //! @post `__other` is in moved-from state. - _CCCL_HOST_API cufile(cufile&& __other) noexcept - : cufile_ref{::cuda::std::exchange(__other.__cufile_handle_, nullptr)} - , __native_handle_{::cuda::std::exchange(__other.__native_handle_, __invalid_native_handle)} - {} - - cufile& operator=(const cufile&) = delete; - - //! @brief Move-assign from a @c cufile object. - //! - //! @param __other The other @c cufile. - //! - //! @post `__other` is in moved-from state. - //! - //! @throws cuda::std::runtime_error if the currently opened file fails to close. - _CCCL_HOST_API cufile& operator=(cufile&& __other) - { - if (this != ::cuda::std::addressof(__other)) - { - close(); - __native_handle_ = ::cuda::std::exchange(__other.__native_handle_, __invalid_native_handle); - __cufile_handle_ = ::cuda::std::exchange(__other.__cufile_handle_, nullptr); - } - return *this; - } - - //! @brief Destructor. Deregisters the cuFile file handle and closes the native handle. - _CCCL_HOST_API ~cufile() - { - if (is_open()) - { - cufile_driver.deregister_native_handle(__cufile_handle_); - [[maybe_unused]] const auto __ignore_close_retval = __close_file_no_throw(__native_handle_); - } - } - - //! @brief Queries whether the file is opened. - //! - //! @return True, if opened, false otherwise. - [[nodiscard]] _CCCL_HOST_API bool is_open() const noexcept - { - return __native_handle_ != __invalid_native_handle; - } - - //! @brief Queries the open mode the object was opened with. - //! - //! @return The \c cuda::cufile_open_mode value if opened, empty value otherwise. - [[nodiscard]] _CCCL_HOST_API cufile_open_mode open_mode() const - { - return is_open() ? __open_mode(__native_handle_) : cufile_open_mode{}; - } - - //! @brief Opens file @c __filename in mode @c __open_mode. - //! - //! @param __filename Path to the file. - //! @param __open_mode Open mode to open the file with. - //! - //! @throws cuda::std::runtime_error if the file cannot be opened or if a file is already opened. - //! @throws cuda::cuda_error if a CUDA driver error occurs. - //! @throws cuda::cufile_error if a cuFile driver error occurs. - _CCCL_HOST_API void open(const char* __filename, cufile_open_mode __open_mode) - { - if (is_open()) - { - _CCCL_THROW(::std::runtime_error, "File is already opened."); - } - - __native_handle_ = __open_file(__filename, __make_oflags(__open_mode)); - - try - { - __cufile_handle_ = cufile_driver.register_native_handle(__native_handle_).get(); - } - catch (...) - { - __close_file(::cuda::std::exchange(__native_handle_, __invalid_native_handle)); - throw; - } - } - - //! @brief Closes the currently opened file. If there is no opened file, no action is taken. - //! - //! @throws cuda::std::runtime_error if the file fails to close. - //! @throws cuda::cuda_error if a CUDA driver error occurs. - //! @throws cuda::cufile_error if a cuFile driver error occurs. - _CCCL_HOST_API void close() - { - if (!is_open()) - { - return; - } - - cufile_driver.deregister_native_handle(::cuda::std::exchange(__cufile_handle_, nullptr)); - __close_file(::cuda::std::exchange(__native_handle_, __invalid_native_handle)); - } - - //! @brief Gets the OS native handle. - //! - //! @return The native handle. - [[nodiscard]] _CCCL_HOST_API native_handle_type native_handle() const noexcept - { - return __native_handle_; - } - - //! @brief Deregisters the cuFile file handle and releases the native handle. The ownership of the native handle is - //! transferred to the caller. - //! - //! @returns The native handle. - [[nodiscard]] _CCCL_HOST_API native_handle_type release() noexcept - { - cufile_driver.deregister_native_handle(::cuda::std::exchange(__cufile_handle_, nullptr)); - return ::cuda::std::exchange(__native_handle_, __invalid_native_handle); - } -}; -} // namespace cuda::experimental diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cufile/cufile_ref.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cufile/cufile_ref.cuh deleted file mode 100644 index 9243c3c1..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cufile/cufile_ref.cuh +++ /dev/null @@ -1,61 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include - -#include - -namespace cuda::experimental -{ -using __cufile_os_native_type = int; - -//! @brief A non-owning wrapper of \c CUfileHandle_t. -class cufile_ref -{ -protected: - ::CUfileHandle_t __cufile_handle_{}; //!< The cuFile file handle. - - _CCCL_HIDE_FROM_ABI cufile_ref() noexcept = default; - -public: - using off_type = ::off_t; - - //! @brief Constructs the object from a \c CUfileHandle_t handle. - _CCCL_HOST_API cufile_ref(::CUfileHandle_t __cufile_handle) noexcept - : __cufile_handle_{__cufile_handle} - {} - - //! @brief Disallow construction from nullptr. - cufile_ref(::cuda::std::nullptr_t) = delete; - - _CCCL_HIDE_FROM_ABI cufile_ref(const cufile_ref&) noexcept = default; - - _CCCL_HIDE_FROM_ABI cufile_ref& operator=(const cufile_ref&) noexcept = default; - - //! @brief Retrieve the \c CUfileHandle_t handle. - //! - //! @returns The handle being held by the object. - [[nodiscard]] _CCCL_HOST_API ::CUfileHandle_t get() const noexcept - { - return __cufile_handle_; - } -}; -} // namespace cuda::experimental diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cufile/driver.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cufile/driver.cuh deleted file mode 100644 index f07d3dba..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cufile/driver.cuh +++ /dev/null @@ -1,314 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include - -#include -#include -#include - -#include - -namespace cuda::experimental -{ -#if _CCCL_CTK_AT_LEAST(13, 0) -//! @brief Structure representing the range of valid values for a cuFile driver attribute. -//! -//! @tparam _Attr The attribute type. Must be one of the types defined in cufile_driver_attributes that has a queryable -//! range. -template -struct cufile_driver_attribute_range -{ - static_assert(_Attr::__has_queryable_range, "Attribute does not have a queryable range"); - - typename _Attr::type min; //!< Minimum value of the attribute. - typename _Attr::type max; //!< Maximum value of the attribute. -}; -#endif // _CCCL_CTK_AT_LEAST(13, 0) - -//! @brief Implementation defined type that implements the cuFILE driver interface. -class cufile_driver_t -{ - _CCCL_HIDE_FROM_ABI constexpr cufile_driver_t() noexcept = default; - -public: - [[nodiscard]] static _CCCL_HOST_API constexpr cufile_driver_t __make_instance() noexcept - { - return cufile_driver_t{}; - } - - cufile_driver_t(const cufile_driver_t&) = delete; - cufile_driver_t& operator=(const cufile_driver_t&) = delete; - cufile_driver_t(cufile_driver_t&&) = delete; - cufile_driver_t& operator=(cufile_driver_t&&) = delete; - - //! @brief Check if the driver is open. - //! - //! @return true if the driver is open, false otherwise. - [[nodiscard]] _CCCL_HOST_API bool is_open() const noexcept - { - return ::cuFileUseCount() > 0; - } - - //! @brief Open the cuFile driver if it is not already open. - //! - //! @throws cufile_error if cuFileDriverOpen fails. - //! @throws cuda_error if a CUDA driver error occurs. - //! - //! @note Some driver attributes cannot be modified after the driver is opened. - //! Attempting to modify these attributes after the driver is opened will result in a runtime error. - _CCCL_HOST_API void open() const - { - if (!is_open()) - { - _CCCL_TRY_CUFILE_API(::cuFileDriverOpen, "Failed to open cuFile driver"); - } - } - - //! @brief Close the cuFile driver if it is open. - //! - //! @throws cufile_error if cuFileDriverClose fails. - //! @throws cuda_error if a CUDA driver error occurs. - _CCCL_HOST_API void close() const - { - if (is_open()) - { - _CCCL_TRY_CUFILE_API(::cuFileDriverClose, "Failed to close cuFile driver"); - } - } - - //! @brief Get the value of a cuFile driver attribute. - //! - //! @tparam _Attr The attribute type to query. Must be one of the types defined in cufile_driver_attributes. - //! - //! @param __attr The attribute to query. - //! - //! @return The value of the attribute. - //! - //! @throws cufile_error if the underlying cuFile API call fails. - //! @throws cuda_error if a CUDA driver error occurs. - //! @throws std::runtime_error if the driver is not open when querying certain attributes. - //! - //! @note Some attributes can only be queried when the driver is open. Attempting to query these attributes - //! when the driver is not open will result in a runtime error. See attribute documentation for details. - template - [[nodiscard]] _CCCL_HOST_API typename _Attr::type attribute([[maybe_unused]] const _Attr& __attr) const - { - using _AttrEnum = typename _Attr::__enum_type; - - typename _Attr::type __ret{}; - if constexpr (::cuda::std::is_same_v<_AttrEnum, ::CUFileSizeTConfigParameter_t>) - { - _CCCL_TRY_CUFILE_API(::cuFileGetParameterSizeT, "Failed to get cuFile parameter", _Attr::__enum_value, &__ret); - } - else if constexpr (::cuda::std::is_same_v<_AttrEnum, ::CUFileBoolConfigParameter_t>) - { - _CCCL_TRY_CUFILE_API(::cuFileGetParameterBool, "Failed to get cuFile parameter", _Attr::__enum_value, &__ret); - } - else if constexpr (::cuda::std::is_same_v<_AttrEnum, ::CUfileDriverStatusFlags_t> - || ::cuda::std::is_same_v<_AttrEnum, ::CUfileFeatureFlags_t>) - { - if (!is_open()) - { - _CCCL_THROW(::std::runtime_error, "cuFile driver must be opened to query this attribute."); - } - - ::CUfileDrvProps_t __props{}; - _CCCL_TRY_CUFILE_API(::cuFileDriverGetProperties, "Failed to get cuFile driver properties", &__props); - - if constexpr (::cuda::std::is_same_v<_AttrEnum, ::CUfileDriverStatusFlags_t>) - { - __ret = __props.nvfs.dstatusflags & _Attr::__enum_value; - } - else - { - __ret = __props.fflags & _Attr::__enum_value; - } - } - else - { - static_assert(::cuda::std::__always_false_v<_AttrEnum>, "Unsupported parameter type"); - } - return __ret; - } - -#if _CCCL_CTK_AT_LEAST(13, 0) - //! @brief Get the valid range of values for a cuFile driver attribute. - //! - //! @tparam _Attr The attribute type to query. Must be one of the types defined in cufile_driver_attributes that has - //! a queryable range. - //! - //! @param __attr The attribute to query. - //! - //! @return The valid range of values for the attribute. - //! - //! @throws cufile_error if the underlying cuFile API call fails. - //! @throws cuda_error if a CUDA driver error occurs. - template - [[nodiscard]] _CCCL_HOST_API cufile_driver_attribute_range<_Attr> - attribute_range([[maybe_unused]] const _Attr& __attr) const - { - static_assert(_Attr::__has_queryable_range, "Attribute does not have a queryable range"); - - using _AttrEnum = typename _Attr::__enum_type; - - if constexpr (::cuda::std::is_same_v<_Attr, cufile_driver_attributes::max_device_cache_size_kb_t> - || ::cuda::std::is_same_v<_Attr, cufile_driver_attributes::max_device_pinned_mem_size_kb_t>) - { - if (!is_open()) - { - _CCCL_THROW(::std::runtime_error, - "This cuFile driver attribute range must be queried after the driver is opened."); - } - } - - cufile_driver_attribute_range<_Attr> __ret{}; - if constexpr (::cuda::std::is_same_v<_AttrEnum, ::CUFileSizeTConfigParameter_t>) - { - _CCCL_TRY_CUFILE_API( - ::cuFileGetParameterMinMaxValue, - "Failed to get cuFile parameter range", - _Attr::__enum_value, - &__ret.min, - &__ret.max); - } - else - { - static_assert(::cuda::std::__always_false_v<_AttrEnum>, "Unsupported parameter type"); - } - return __ret; - } -#endif // _CCCL_CTK_AT_LEAST(13, 0) - - //! @brief Set the value of a cuFile driver attribute. - //! - //! @tparam _Attr The attribute type to set. Must be one of the types defined in cufile_driver_attributes that is not - //! read-only. - //! - //! @param __attr The attribute to set. - //! @param __value The value to set the attribute to. - //! - //! @throws cufile_error if the underlying cuFile API call fails. - //! @throws cuda_error if a CUDA driver error occurs. - //! @throws std::runtime_error if the attribute cannot be modified after the driver is opened. - //! - //! @note Some attributes cannot be modified after the driver is opened. Attempting to modify these attributes - //! after the driver is opened will result in a runtime error. See attribute documentation for details. - template - _CCCL_HOST_API void set_attribute([[maybe_unused]] const _Attr& __attr, typename _Attr::type __value) const - { - static_assert(_Attr::__can_be_set_when_closed || _Attr::__can_be_set_when_opened, - "Cannot modify read-only attribute"); - - using _AttrEnum = typename _Attr::__enum_type; - - if (is_open()) - { - if constexpr (::cuda::std::is_same_v<_Attr, cufile_driver_attributes::use_poll_mode_t>) - { - const auto __pollthreshold_size = attribute(cufile_driver_attributes::pollthreshold_size_kb); - _CCCL_TRY_CUFILE_API( - ::cuFileDriverSetPollMode, "Failed to set cuFile driver poll mode", __value, __pollthreshold_size); - } - else if constexpr (::cuda::std::is_same_v<_Attr, cufile_driver_attributes::pollthreshold_size_kb_t>) - { - const auto __use_poll_mode = attribute(cufile_driver_attributes::use_poll_mode); - _CCCL_TRY_CUFILE_API( - ::cuFileDriverSetPollMode, "Failed to set cuFile driver poll mode", __use_poll_mode, __value); - } - else if constexpr (::cuda::std::is_same_v<_Attr, cufile_driver_attributes::max_direct_io_size_kb_t>) - { - _CCCL_TRY_CUFILE_API( - ::cuFileDriverSetMaxDirectIOSize, "Failed to set cuFile driver max direct IO size", __value); - } - else if constexpr (::cuda::std::is_same_v<_Attr, cufile_driver_attributes::max_device_cache_size_kb_t>) - { - _CCCL_TRY_CUFILE_API(::cuFileDriverSetMaxCacheSize, "Failed to set cuFile driver max cache size", __value); - } - else if constexpr (::cuda::std::is_same_v<_Attr, cufile_driver_attributes::max_device_pinned_mem_size_kb_t>) - { - _CCCL_TRY_CUFILE_API( - ::cuFileDriverSetMaxPinnedMemSize, "Failed to set cuFile driver max pinned mem size", __value); - } - else - { - _CCCL_THROW(::std::runtime_error, - "This cuFile driver attribute cannot be modified after the driver is opened."); - } - } - else - { - if constexpr (::cuda::std::is_same_v<_AttrEnum, ::CUFileSizeTConfigParameter_t>) - { - _CCCL_TRY_CUFILE_API( - ::cuFileSetParameterSizeT, "Failed to set cuFile parameter size", _Attr::__enum_value, __value); - } - else if constexpr (::cuda::std::is_same_v<_AttrEnum, ::CUFileBoolConfigParameter_t>) - { - _CCCL_TRY_CUFILE_API( - ::cuFileSetParameterBool, "Failed to set cuFile parameter bool", _Attr::__enum_value, __value); - } - else - { - static_assert(::cuda::std::__always_false_v<_AttrEnum>, "Unsupported parameter type"); - } - } - } - - //! @brief Registers an OS native file handle type in the cuFile driver. The registered cuFile handle can be used with - //! other cuFile APIs. The handle must be deregistered calling the \c - //! cuda::cufile_driver.deregister_native_handle(...) method with the obtained cuFile handle. For each OS - //! native file handle can be called once before deregistered. - //! - //! @param __native_handle The OS native file handle. - //! - //! @return \c cuda::cufile_ref handle. - //! - //! @throws cuda::cuda_error if a CUDA driver error occurs. - //! @throws cuda::cufile_error if a cuFile driver error occurs. - [[nodiscard]] _CCCL_HOST_API cufile_ref register_native_handle(__cufile_os_native_type __native_handle) const - { - ::CUfileDescr_t __desc{}; - __desc.type = ::CU_FILE_HANDLE_TYPE_OPAQUE_FD; - __desc.handle.fd = __native_handle; - - ::CUfileHandle_t __handle{}; - _CCCL_TRY_CUFILE_API(::cuFileHandleRegister, "Failed to register cuFile handle", &__handle, &__desc); - return __handle; - } - - //! @brief Deregisters the previously registered cuFile handle in the driver. - //! - //! @param __file The cuFile handle. - //! - //! @note The \c cuda::cufile implementation relies on this function being \c noexcept. - _CCCL_HOST_API void deregister_native_handle(cufile_ref __file) const noexcept - { - ::cuFileHandleDeregister(__file.get()); - } -}; - -//! @brief Global instance of the cuFile driver interface. -inline constexpr cufile_driver_t cufile_driver = cufile_driver_t::__make_instance(); -} // namespace cuda::experimental diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cufile/driver_attributes.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cufile/driver_attributes.cuh deleted file mode 100644 index 5eec23bf..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cufile/driver_attributes.cuh +++ /dev/null @@ -1,173 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include - -#include - -namespace cuda::experimental::cufile_driver_attributes -{ -template -[[nodiscard]] _CCCL_CONSTEVAL auto __attr_from_param_type() noexcept -{ - if constexpr (::cuda::std::is_same_v<_ParamEnum, ::CUFileSizeTConfigParameter_t>) - { - return ::cuda::std::size_t{}; - } - else if constexpr (::cuda::std::is_same_v<_ParamEnum, ::CUFileBoolConfigParameter_t>) - { - return bool{}; - } - else - { - static_assert(::cuda::std::__always_false_v<_ParamEnum>, "Unsupported parameter type"); - } -} - -template -struct __attr_from_param -{ - using __enum_type = decltype(_Param); - static constexpr auto __enum_value = _Param; - static constexpr auto __can_be_get_when_closed = true; - static constexpr auto __can_be_set_when_closed = true; - static constexpr auto __can_be_set_when_opened = _CanBeSetWhenOpened; - static constexpr auto __has_queryable_range = ::cuda::std::is_same_v<__enum_type, ::CUFileSizeTConfigParameter_t>; - - using type = decltype(__attr_from_param_type<__enum_type>()); -}; - -template <::CUfileDriverStatusFlags_t _Status> -struct __attr_from_status -{ - using __enum_type = ::CUfileDriverStatusFlags_t; - static constexpr auto __enum_value = _Status; - static constexpr auto __can_be_get_when_closed = false; - static constexpr auto __can_be_set_when_closed = false; - static constexpr auto __can_be_set_when_opened = false; - static constexpr auto __has_queryable_range = false; - - using type = bool; -}; - -template <::CUfileFeatureFlags_t _Feature> -struct __attr_from_feature -{ - using __enum_type = ::CUfileFeatureFlags_t; - static constexpr auto __enum_value = _Feature; - static constexpr auto __can_be_get_when_closed = false; - static constexpr auto __can_be_set_when_closed = false; - static constexpr auto __can_be_set_when_opened = false; - static constexpr auto __has_queryable_range = false; - - using type = bool; -}; - -using max_io_queue_depth_t = __attr_from_param<::CUFILE_PARAM_EXECUTION_MAX_IO_QUEUE_DEPTH>; -using max_io_threads_t = __attr_from_param<::CUFILE_PARAM_EXECUTION_MAX_IO_THREADS>; -using min_io_threshold_size_kb_t = __attr_from_param<::CUFILE_PARAM_EXECUTION_MIN_IO_THRESHOLD_SIZE_KB>; -using max_request_parallelism_t = __attr_from_param<::CUFILE_PARAM_EXECUTION_MAX_REQUEST_PARALLELISM>; -using max_direct_io_size_kb_t = __attr_from_param<::CUFILE_PARAM_PROPERTIES_MAX_DIRECT_IO_SIZE_KB, true>; -using max_device_cache_size_kb_t = __attr_from_param<::CUFILE_PARAM_PROPERTIES_MAX_DEVICE_CACHE_SIZE_KB, true>; -using per_buffer_cache_size_kb_t = __attr_from_param<::CUFILE_PARAM_PROPERTIES_PER_BUFFER_CACHE_SIZE_KB>; -using max_device_pinned_mem_size_kb_t = - __attr_from_param<::CUFILE_PARAM_PROPERTIES_MAX_DEVICE_PINNED_MEM_SIZE_KB, true>; -using io_batchsize_t = __attr_from_param<::CUFILE_PARAM_PROPERTIES_IO_BATCHSIZE>; -using pollthreshold_size_kb_t = __attr_from_param<::CUFILE_PARAM_POLLTHRESHOLD_SIZE_KB, true>; -using batch_io_timeout_ms_t = __attr_from_param<::CUFILE_PARAM_PROPERTIES_BATCH_IO_TIMEOUT_MS>; -using use_poll_mode_t = __attr_from_param<::CUFILE_PARAM_PROPERTIES_USE_POLL_MODE, true>; -using allow_compat_mode_t = __attr_from_param<::CUFILE_PARAM_PROPERTIES_ALLOW_COMPAT_MODE>; -using force_compat_mode_t = __attr_from_param<::CUFILE_PARAM_FORCE_COMPAT_MODE>; -using fs_misc_api_check_aggressive_t = __attr_from_param<::CUFILE_PARAM_FS_MISC_API_CHECK_AGGRESSIVE>; -using parallel_io_t = __attr_from_param<::CUFILE_PARAM_EXECUTION_PARALLEL_IO>; -using profile_nvtx_t = __attr_from_param<::CUFILE_PARAM_PROFILE_NVTX>; -using allow_system_memory_t = __attr_from_param<::CUFILE_PARAM_PROPERTIES_ALLOW_SYSTEM_MEMORY>; -using use_pcip2pdma_t = __attr_from_param<::CUFILE_PARAM_USE_PCIP2PDMA>; -using prefer_io_uring_t = __attr_from_param<::CUFILE_PARAM_PREFER_IO_URING>; -using force_odirect_mode_t = __attr_from_param<::CUFILE_PARAM_FORCE_ODIRECT_MODE>; -using skip_topology_detection_t = __attr_from_param<::CUFILE_PARAM_SKIP_TOPOLOGY_DETECTION>; -using stream_memops_bypass_t = __attr_from_param<::CUFILE_PARAM_STREAM_MEMOPS_BYPASS>; -using has_luster_support_t = __attr_from_status<::CU_FILE_LUSTRE_SUPPORTED>; -using has_wekafs_support_t = __attr_from_status<::CU_FILE_WEKAFS_SUPPORTED>; -using has_nfs_support_t = __attr_from_status<::CU_FILE_NFS_SUPPORTED>; -using has_gpfs_support_t = __attr_from_status<::CU_FILE_GPFS_SUPPORTED>; -using has_nvme_support_t = __attr_from_status<::CU_FILE_NVME_SUPPORTED>; -using has_nvmeof_support_t = __attr_from_status<::CU_FILE_NVMEOF_SUPPORTED>; -using has_scsi_support_t = __attr_from_status<::CU_FILE_SCSI_SUPPORTED>; -using has_scaleflux_csd_support_t = __attr_from_status<::CU_FILE_SCALEFLUX_CSD_SUPPORTED>; -using has_nvmesh_support_t = __attr_from_status<::CU_FILE_NVMESH_SUPPORTED>; -using has_beegfs_support_t = __attr_from_status<::CU_FILE_BEEGFS_SUPPORTED>; -using has_nvme_p2p_support_t = __attr_from_status<::CU_FILE_NVME_P2P_SUPPORTED>; -using has_scatefs_support_t = __attr_from_status<::CU_FILE_SCATEFS_SUPPORTED>; -using has_dynamic_routing_support_t = __attr_from_feature<::CU_FILE_DYN_ROUTING_SUPPORTED>; -using has_batch_io_support_t = __attr_from_feature<::CU_FILE_BATCH_IO_SUPPORTED>; -using has_streams_support_t = __attr_from_feature<::CU_FILE_STREAMS_SUPPORTED>; -using has_parallel_io_support_t = __attr_from_feature<::CU_FILE_PARALLEL_IO_SUPPORTED>; - -// todo: add documentation of each attribute -// 1. type -// 2. whether it is read-only or can be set -// 3. if it can be set/read when driver is open/closed -// 4. default value, constraints - -inline constexpr max_io_queue_depth_t max_io_queue_depth{}; -inline constexpr max_io_threads_t max_io_threads{}; -inline constexpr min_io_threshold_size_kb_t min_io_threshold_size_kb{}; -inline constexpr max_request_parallelism_t max_request_parallelism{}; -inline constexpr max_direct_io_size_kb_t max_direct_io_size_kb{}; -inline constexpr max_device_cache_size_kb_t max_device_cache_size_kb{}; -inline constexpr per_buffer_cache_size_kb_t per_buffer_cache_size_kb{}; -inline constexpr max_device_pinned_mem_size_kb_t max_device_pinned_mem_size_kb{}; -inline constexpr io_batchsize_t io_batchsize{}; -inline constexpr pollthreshold_size_kb_t pollthreshold_size_kb{}; -inline constexpr batch_io_timeout_ms_t batch_io_timeout_ms{}; -inline constexpr use_poll_mode_t use_poll_mode{}; -inline constexpr allow_compat_mode_t allow_compat_mode{}; -inline constexpr force_compat_mode_t force_compat_mode{}; -inline constexpr fs_misc_api_check_aggressive_t fs_misc_api_check_aggressive{}; -inline constexpr parallel_io_t parallel_io{}; -inline constexpr profile_nvtx_t profile_nvtx{}; -inline constexpr allow_system_memory_t allow_system_memory{}; -inline constexpr use_pcip2pdma_t use_pcip2pdma{}; -inline constexpr prefer_io_uring_t prefer_io_uring{}; -inline constexpr force_odirect_mode_t force_odirect_mode{}; -inline constexpr skip_topology_detection_t skip_topology_detection{}; -inline constexpr stream_memops_bypass_t stream_memops_bypass{}; -inline constexpr has_luster_support_t has_luster_support{}; -inline constexpr has_wekafs_support_t has_wekafs_support{}; -inline constexpr has_nfs_support_t has_nfs_support{}; -inline constexpr has_gpfs_support_t has_gpfs_support{}; -inline constexpr has_nvme_support_t has_nvme_support{}; -inline constexpr has_nvmeof_support_t has_nvmeof_support{}; -inline constexpr has_scsi_support_t has_scsi_support{}; -inline constexpr has_scaleflux_csd_support_t has_scaleflux_csd_support{}; -inline constexpr has_nvmesh_support_t has_nvmesh_support{}; -inline constexpr has_beegfs_support_t has_beegfs_support{}; -inline constexpr has_nvme_p2p_support_t has_nvme_p2p_support{}; -inline constexpr has_scatefs_support_t has_scatefs_support{}; -inline constexpr has_dynamic_routing_support_t has_dynamic_routing_support{}; -inline constexpr has_batch_io_support_t has_batch_io_support{}; -inline constexpr has_streams_support_t has_streams_support{}; -inline constexpr has_parallel_io_support_t has_parallel_io_support{}; -} // namespace cuda::experimental::cufile_driver_attributes diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cufile/exception.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cufile/exception.cuh deleted file mode 100644 index db8d8822..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cufile/exception.cuh +++ /dev/null @@ -1,107 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include - -#include - -#include - -namespace cuda::experimental -{ -#if _CCCL_HAS_CTK() -using __cufile_error_t = ::CUfileOpError; -#else // ^^^ _CCCL_HAS_CTK() ^^^ // vvv !_CCCL_HAS_CTK() vvv -using __cufile_error_t = int; -#endif // ^^^ !_CCCL_HAS_CTK() ^^^ - -struct __cufile_msg_storage -{ - char __buffer[512]{}; -}; - -static char* __format_cufile_error_message( - __cufile_msg_storage& __msg_buffer, - const __cufile_error_t __status, - const char* __msg, - const char* __api = nullptr, - ::cuda::std::source_location __loc = ::cuda::std::source_location::current()) noexcept -{ - ::snprintf( - __msg_buffer.__buffer, - 512, - "%s:%d %s%s%s(%d): %s", - __loc.file_name(), - __loc.line(), - __api ? __api : "", - __api ? " " : "", -#if _CCCL_HAS_CTK() - ::cufileop_status_error(::CUfileOpError{__status}), -#else // ^^^ _CCCL_HAS_CTK() ^^^ / vvv !_CCCL_HAS_CTK() vvv - "cuFile error", -#endif // ^^^ !_CCCL_HAS_CTK() ^^^ - __status, - __msg); - return __msg_buffer.__buffer; -} - -//! @brief Exception class for errors from cuFile APIs. -class cufile_error : public ::std::runtime_error -{ - __cufile_error_t __status_; //!< The cuFile error code. - -public: - _CCCL_HOST_API cufile_error( - __cufile_error_t __status, - const char* __msg, - const char* __api, - ::cuda::std::source_location loc = ::cuda::std::source_location::current(), - __cufile_msg_storage __msg_buffer = {}) - : ::std::runtime_error{__format_cufile_error_message(__msg_buffer, __status, __msg, __api, loc)} - , __status_{__status} - {} - - [[nodiscard]] _CCCL_HOST_API __cufile_error_t status() const noexcept - { - return __status_; - } -}; - -//! @brief Macro to call a cuFile API and throw a cufile_error or cuda_error if it fails. -#define _CCCL_TRY_CUFILE_API(_NAME, _MSG, ...) \ - do \ - { \ - const ::CUfileError_t __cufile_error_status = _NAME(__VA_ARGS__); \ - switch (__cufile_error_status.err) \ - { \ - case ::CU_FILE_SUCCESS: \ - break; \ - case ::CU_FILE_CUDA_DRIVER_ERROR: \ - _CCCL_THROW(::cuda::cuda_error, static_cast<::cudaError_t>(__cufile_error_status.cu_err), _MSG, #_NAME); \ - default: \ - _CCCL_THROW(::cuda::experimental::cufile_error, __cufile_error_status.err, _MSG, #_NAME); \ - } \ - } while (0) -} // namespace cuda::experimental diff --git a/cccl_upstream/cudax/include/cuda/experimental/__cufile/open_mode.cuh b/cccl_upstream/cudax/include/cuda/experimental/__cufile/open_mode.cuh deleted file mode 100644 index aa3c0629..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__cufile/open_mode.cuh +++ /dev/null @@ -1,73 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include - -namespace cuda::experimental -{ -//! @brief Open mode for cufile. -enum class cufile_open_mode : unsigned -{ - in = (1u << 0), - out = (1u << 1), - trunc = (1u << 2), - noreplace = (1u << 3), - direct = (1u << 4), -}; - -[[nodiscard]] _CCCL_HOST_API constexpr cufile_open_mode -operator|(cufile_open_mode __lhs, cufile_open_mode __rhs) noexcept -{ - return static_cast(::cuda::std::to_underlying(__lhs) | ::cuda::std::to_underlying(__rhs)); -} - -_CCCL_HOST_API constexpr cufile_open_mode& operator|=(cufile_open_mode& __lhs, cufile_open_mode __rhs) noexcept -{ - return __lhs = __lhs | __rhs; -} - -[[nodiscard]] _CCCL_HOST_API constexpr cufile_open_mode -operator&(cufile_open_mode __lhs, cufile_open_mode __rhs) noexcept -{ - return static_cast(::cuda::std::to_underlying(__lhs) & ::cuda::std::to_underlying(__rhs)); -} - -_CCCL_HOST_API constexpr cufile_open_mode& operator&=(cufile_open_mode& __lhs, cufile_open_mode __rhs) noexcept -{ - return __lhs = __lhs & __rhs; -} - -[[nodiscard]] _CCCL_HOST_API constexpr cufile_open_mode -operator^(cufile_open_mode __lhs, cufile_open_mode __rhs) noexcept -{ - return static_cast(::cuda::std::to_underlying(__lhs) ^ ::cuda::std::to_underlying(__rhs)); -} - -_CCCL_HOST_API constexpr cufile_open_mode& operator^=(cufile_open_mode& __lhs, cufile_open_mode __rhs) noexcept -{ - return __lhs = __lhs ^ __rhs; -} - -[[nodiscard]] _CCCL_HOST_API constexpr cufile_open_mode operator~(cufile_open_mode __b) noexcept -{ - return static_cast(~::cuda::std::to_underlying(__b)); -} -} // namespace cuda::experimental diff --git a/cccl_upstream/cudax/include/cuda/experimental/__detail/type_traits.cuh b/cccl_upstream/cudax/include/cuda/experimental/__detail/type_traits.cuh deleted file mode 100644 index f75bebe2..00000000 --- a/cccl_upstream/cudax/include/cuda/experimental/__detail/type_traits.cuh +++ /dev/null @@ -1,119 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of CUDA Experimental in CUDA C++ Core Libraries, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef __CUDAX_DETAIL_TYPE_TRAITS_CUH -#define __CUDAX_DETAIL_TYPE_TRAITS_CUH - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace cuda::experimental -{ -using ::cuda::std::__declfn_t; -using ::cuda::std::decay_t; - -template -[[noreturn]] _CCCL_HOST_DEVICE_API auto __declfn() noexcept(_Nothrow) -> _Ty -{ - _CCCL_ASSERT(false, "__declfn should never be called at runtime."); - _CCCL_UNREACHABLE(); -} - -template -_CCCL_CONCEPT __same_as = ::cuda::std::_IsSame<_Ty, _Uy>::value; - -template -_CCCL_CONCEPT __not_same_as = !::cuda::std::_IsSame<_Ty, _Uy>::value; - -template -_CCCL_CONCEPT __one_of = (__same_as<_Ty, _Us> || ...); - -template -_CCCL_CONCEPT __none_of = (__not_same_as<_Ty, _Us> && ...); - -#if _CCCL_HAS_CONCEPTS() - -template