[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
82
cccl_upstream/c/parallel.v2/include/cccl/c/binary_search.h
Normal file
82
cccl_upstream/c/parallel.v2/include/cccl/c/binary_search.h
Normal file
@@ -0,0 +1,82 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
// NOLINTBEGIN(modernize-use-using)
|
||||
|
||||
#ifndef CCCL_C_EXPERIMENTAL
|
||||
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
|
||||
#endif // !CCCL_C_EXPERIMENTAL
|
||||
|
||||
#include <cuda.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <cccl/c/extern_c.h>
|
||||
#include <cccl/c/types.h>
|
||||
|
||||
CCCL_C_EXTERN_C_BEGIN
|
||||
|
||||
typedef struct cccl_device_binary_search_build_result_t
|
||||
{
|
||||
int cc;
|
||||
void* payload;
|
||||
size_t payload_size;
|
||||
void* jit_compiler; // hostjit::JITCompiler*
|
||||
#if defined(_WIN32)
|
||||
// Opaque state for serializing CUB's lazy first-call initialization.
|
||||
void* first_call_state;
|
||||
#endif // _WIN32
|
||||
void* binary_search_fn; // int(*)(void*, ull, void*, ull, void*, void*, void*)
|
||||
} cccl_device_binary_search_build_result_t;
|
||||
|
||||
CCCL_C_API CUresult cccl_device_binary_search_build(
|
||||
cccl_device_binary_search_build_result_t* build,
|
||||
cccl_binary_search_mode_t mode,
|
||||
cccl_iterator_t d_data,
|
||||
cccl_iterator_t d_values,
|
||||
cccl_iterator_t d_out,
|
||||
cccl_op_t op,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path);
|
||||
|
||||
// Extended version with build configuration
|
||||
CCCL_C_API CUresult cccl_device_binary_search_build_ex(
|
||||
cccl_device_binary_search_build_result_t* build,
|
||||
cccl_binary_search_mode_t mode,
|
||||
cccl_iterator_t d_data,
|
||||
cccl_iterator_t d_values,
|
||||
cccl_iterator_t d_out,
|
||||
cccl_op_t op,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path,
|
||||
cccl_build_config* config);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_binary_search(
|
||||
cccl_device_binary_search_build_result_t build,
|
||||
cccl_iterator_t d_data,
|
||||
uint64_t num_items,
|
||||
cccl_iterator_t d_values,
|
||||
uint64_t num_values,
|
||||
cccl_iterator_t d_out,
|
||||
cccl_op_t op,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_binary_search_cleanup(cccl_device_binary_search_build_result_t* bld_ptr);
|
||||
|
||||
CCCL_C_EXTERN_C_END
|
||||
// NOLINTEND(modernize-use-using)
|
||||
23
cccl_upstream/c/parallel.v2/include/cccl/c/extern_c.h
Normal file
23
cccl_upstream/c/parallel.v2/include/cccl/c/extern_c.h
Normal file
@@ -0,0 +1,23 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA Core Compute Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
# define CCCL_C_EXTERN_C_BEGIN extern "C" {
|
||||
# define CCCL_C_EXTERN_C_END }
|
||||
|
||||
#else
|
||||
|
||||
# define CCCL_C_EXTERN_C_BEGIN
|
||||
# define CCCL_C_EXTERN_C_END
|
||||
|
||||
#endif
|
||||
65
cccl_upstream/c/parallel.v2/include/cccl/c/for.h
Normal file
65
cccl_upstream/c/parallel.v2/include/cccl/c/for.h
Normal file
@@ -0,0 +1,65 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
// NOLINTBEGIN(modernize-use-using)
|
||||
|
||||
#ifndef CCCL_C_EXPERIMENTAL
|
||||
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
|
||||
#endif // !CCCL_C_EXPERIMENTAL
|
||||
|
||||
#include <cuda.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <cccl/c/extern_c.h>
|
||||
#include <cccl/c/types.h>
|
||||
|
||||
CCCL_C_EXTERN_C_BEGIN
|
||||
|
||||
typedef struct cccl_device_for_build_result_t
|
||||
{
|
||||
int cc;
|
||||
void* payload;
|
||||
size_t payload_size;
|
||||
void* jit_compiler; // hostjit::JITCompiler*
|
||||
void* for_fn; // int(*)(void*, unsigned long long, void*)
|
||||
} cccl_device_for_build_result_t;
|
||||
|
||||
CCCL_C_API CUresult cccl_device_for_build(
|
||||
cccl_device_for_build_result_t* build,
|
||||
cccl_iterator_t d_data,
|
||||
cccl_op_t op,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path);
|
||||
|
||||
// Extended version with build configuration
|
||||
CCCL_C_API CUresult cccl_device_for_build_ex(
|
||||
cccl_device_for_build_result_t* build,
|
||||
cccl_iterator_t d_data,
|
||||
cccl_op_t op,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path,
|
||||
cccl_build_config* config);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_for(
|
||||
cccl_device_for_build_result_t build, cccl_iterator_t d_data, uint64_t num_items, cccl_op_t op, CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_for_cleanup(cccl_device_for_build_result_t* bld_ptr);
|
||||
|
||||
CCCL_C_EXTERN_C_END
|
||||
// NOLINTEND(modernize-use-using)
|
||||
96
cccl_upstream/c/parallel.v2/include/cccl/c/histogram.h
Normal file
96
cccl_upstream/c/parallel.v2/include/cccl/c/histogram.h
Normal file
@@ -0,0 +1,96 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA Core Compute Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
// NOLINTBEGIN(modernize-use-using)
|
||||
|
||||
#ifndef CCCL_C_EXPERIMENTAL
|
||||
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
|
||||
#endif // !CCCL_C_EXPERIMENTAL
|
||||
|
||||
#include <cuda.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <cccl/c/extern_c.h>
|
||||
#include <cccl/c/types.h>
|
||||
|
||||
CCCL_C_EXTERN_C_BEGIN
|
||||
|
||||
typedef struct cccl_device_histogram_build_result_t
|
||||
{
|
||||
int cc;
|
||||
void* payload;
|
||||
size_t payload_size;
|
||||
void* jit_compiler;
|
||||
void* histogram_fn;
|
||||
cccl_type_info counter_type;
|
||||
cccl_type_info level_type;
|
||||
cccl_type_info sample_type;
|
||||
int num_channels;
|
||||
int num_active_channels;
|
||||
} cccl_device_histogram_build_result_t;
|
||||
|
||||
CCCL_C_API CUresult cccl_device_histogram_build(
|
||||
cccl_device_histogram_build_result_t* build,
|
||||
int num_channels,
|
||||
int num_active_channels,
|
||||
cccl_iterator_t d_samples,
|
||||
int num_output_levels_val,
|
||||
cccl_iterator_t d_output_histograms,
|
||||
cccl_type_info level_type,
|
||||
int64_t num_rows,
|
||||
int64_t row_stride_samples,
|
||||
bool is_evenly_segmented,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path);
|
||||
|
||||
// Extended version with build configuration
|
||||
CCCL_C_API CUresult cccl_device_histogram_build_ex(
|
||||
cccl_device_histogram_build_result_t* build,
|
||||
int num_channels,
|
||||
int num_active_channels,
|
||||
cccl_iterator_t d_samples,
|
||||
int num_output_levels_val,
|
||||
cccl_iterator_t d_output_histograms,
|
||||
cccl_type_info level_type,
|
||||
int64_t num_rows,
|
||||
int64_t row_stride_samples,
|
||||
bool is_evenly_segmented,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path,
|
||||
cccl_build_config* config);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_histogram_even(
|
||||
cccl_device_histogram_build_result_t build,
|
||||
void* d_temp_storage,
|
||||
size_t* temp_storage_bytes,
|
||||
cccl_iterator_t d_samples,
|
||||
cccl_iterator_t d_output_histograms,
|
||||
cccl_value_t num_output_levels,
|
||||
cccl_value_t lower_level,
|
||||
cccl_value_t upper_level,
|
||||
int64_t num_row_pixels,
|
||||
int64_t num_rows,
|
||||
int64_t row_stride_samples,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_histogram_cleanup(cccl_device_histogram_build_result_t* bld_ptr);
|
||||
|
||||
CCCL_C_EXTERN_C_END
|
||||
// NOLINTEND(modernize-use-using)
|
||||
86
cccl_upstream/c/parallel.v2/include/cccl/c/merge_sort.h
Normal file
86
cccl_upstream/c/parallel.v2/include/cccl/c/merge_sort.h
Normal file
@@ -0,0 +1,86 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA Core Compute Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
// NOLINTBEGIN(modernize-use-using)
|
||||
|
||||
#ifndef CCCL_C_EXPERIMENTAL
|
||||
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
|
||||
#endif // !CCCL_C_EXPERIMENTAL
|
||||
|
||||
#include <cuda.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <cccl/c/extern_c.h>
|
||||
#include <cccl/c/types.h>
|
||||
|
||||
CCCL_C_EXTERN_C_BEGIN
|
||||
|
||||
typedef struct cccl_device_merge_sort_build_result_t
|
||||
{
|
||||
int cc;
|
||||
void* payload;
|
||||
size_t payload_size;
|
||||
void* jit_compiler;
|
||||
void* sort_fn;
|
||||
// 1 if the build compiled SortKeysCopy (no items), 0 if SortPairsCopy. The
|
||||
// run function dispatches on this so the value-vs-pairs decision doesn't
|
||||
// have to be re-derived from the iterator arguments.
|
||||
int keys_only;
|
||||
cccl_type_info key_type;
|
||||
cccl_type_info item_type;
|
||||
} cccl_device_merge_sort_build_result_t;
|
||||
|
||||
CCCL_C_API CUresult cccl_device_merge_sort_build(
|
||||
cccl_device_merge_sort_build_result_t* build,
|
||||
cccl_iterator_t d_in_keys,
|
||||
cccl_iterator_t d_in_items,
|
||||
cccl_iterator_t d_out_keys,
|
||||
cccl_iterator_t d_out_items,
|
||||
cccl_op_t op,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path);
|
||||
|
||||
// Extended version with build configuration
|
||||
CCCL_C_API CUresult cccl_device_merge_sort_build_ex(
|
||||
cccl_device_merge_sort_build_result_t* build,
|
||||
cccl_iterator_t d_in_keys,
|
||||
cccl_iterator_t d_in_items,
|
||||
cccl_iterator_t d_out_keys,
|
||||
cccl_iterator_t d_out_items,
|
||||
cccl_op_t op,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path,
|
||||
cccl_build_config* config);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_merge_sort(
|
||||
cccl_device_merge_sort_build_result_t build,
|
||||
void* d_temp_storage,
|
||||
size_t* temp_storage_bytes,
|
||||
cccl_iterator_t d_in_keys,
|
||||
cccl_iterator_t d_in_items,
|
||||
cccl_iterator_t d_out_keys,
|
||||
cccl_iterator_t d_out_items,
|
||||
uint64_t num_items,
|
||||
cccl_op_t op,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_merge_sort_cleanup(cccl_device_merge_sort_build_result_t* bld_ptr);
|
||||
|
||||
CCCL_C_EXTERN_C_END
|
||||
// NOLINTEND(modernize-use-using)
|
||||
90
cccl_upstream/c/parallel.v2/include/cccl/c/radix_sort.h
Normal file
90
cccl_upstream/c/parallel.v2/include/cccl/c/radix_sort.h
Normal file
@@ -0,0 +1,90 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA Core Compute Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
// NOLINTBEGIN(modernize-use-using)
|
||||
|
||||
#ifndef CCCL_C_EXPERIMENTAL
|
||||
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
|
||||
#endif // !CCCL_C_EXPERIMENTAL
|
||||
|
||||
#include <cuda.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <cccl/c/extern_c.h>
|
||||
#include <cccl/c/types.h>
|
||||
|
||||
CCCL_C_EXTERN_C_BEGIN
|
||||
|
||||
typedef struct cccl_device_radix_sort_build_result_t
|
||||
{
|
||||
int cc;
|
||||
void* payload;
|
||||
size_t payload_size;
|
||||
void* jit_compiler; /* Owns both wrappers below — one TU, one cubin */
|
||||
void* sort_fn; /* Wrapper around CUB's copy-overload (selector always 0) */
|
||||
void* sort_fn_overwrite; /* Wrapper around CUB's DoubleBuffer overload; reports selector */
|
||||
cccl_type_info key_type;
|
||||
cccl_type_info value_type;
|
||||
cccl_sort_order_t order;
|
||||
int keys_only; /* 1 if keys-only sort, 0 if key-value pairs */
|
||||
} cccl_device_radix_sort_build_result_t;
|
||||
|
||||
CCCL_C_API CUresult cccl_device_radix_sort_build(
|
||||
cccl_device_radix_sort_build_result_t* build,
|
||||
cccl_sort_order_t sort_order,
|
||||
cccl_iterator_t input_keys_it,
|
||||
cccl_iterator_t input_values_it,
|
||||
cccl_op_t decomposer,
|
||||
const char* decomposer_return_type,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path);
|
||||
|
||||
// Extended version with build configuration
|
||||
CCCL_C_API CUresult cccl_device_radix_sort_build_ex(
|
||||
cccl_device_radix_sort_build_result_t* build,
|
||||
cccl_sort_order_t sort_order,
|
||||
cccl_iterator_t input_keys_it,
|
||||
cccl_iterator_t input_values_it,
|
||||
cccl_op_t decomposer,
|
||||
const char* decomposer_return_type,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path,
|
||||
cccl_build_config* config);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_radix_sort(
|
||||
cccl_device_radix_sort_build_result_t build,
|
||||
void* d_temp_storage,
|
||||
size_t* temp_storage_bytes,
|
||||
cccl_iterator_t d_keys_in,
|
||||
cccl_iterator_t d_keys_out,
|
||||
cccl_iterator_t d_values_in,
|
||||
cccl_iterator_t d_values_out,
|
||||
cccl_op_t decomposer,
|
||||
uint64_t num_items,
|
||||
int begin_bit,
|
||||
int end_bit,
|
||||
bool is_overwrite_okay,
|
||||
int* selector,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_radix_sort_cleanup(cccl_device_radix_sort_build_result_t* bld_ptr);
|
||||
|
||||
CCCL_C_EXTERN_C_END
|
||||
// NOLINTEND(modernize-use-using)
|
||||
94
cccl_upstream/c/parallel.v2/include/cccl/c/reduce.h
Normal file
94
cccl_upstream/c/parallel.v2/include/cccl/c/reduce.h
Normal file
@@ -0,0 +1,94 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA Core Compute Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
// NOLINTBEGIN(modernize-use-using)
|
||||
|
||||
#ifndef CCCL_C_EXPERIMENTAL
|
||||
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
|
||||
#endif // !CCCL_C_EXPERIMENTAL
|
||||
|
||||
#include <cuda.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <cccl/c/extern_c.h>
|
||||
#include <cccl/c/types.h>
|
||||
|
||||
CCCL_C_EXTERN_C_BEGIN
|
||||
|
||||
typedef struct cccl_device_reduce_build_result_t
|
||||
{
|
||||
int cc;
|
||||
void* payload;
|
||||
size_t payload_size;
|
||||
void* jit_compiler; // hostjit::JITCompiler*
|
||||
void* reduce_fn; // int(*)(void*, size_t*, void*, void*, unsigned long long, void*, void*, void*) — trailing void* is
|
||||
// the CUstream
|
||||
uint64_t accumulator_size;
|
||||
cccl_determinism_t determinism;
|
||||
} cccl_device_reduce_build_result_t;
|
||||
|
||||
// TODO return a union of nvtx/cuda/nvrtc errors or a string?
|
||||
CCCL_C_API CUresult cccl_device_reduce_build(
|
||||
cccl_device_reduce_build_result_t* build,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
cccl_op_t op,
|
||||
cccl_value_t init,
|
||||
cccl_determinism_t determinism,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path);
|
||||
|
||||
// Extended version with build configuration
|
||||
CCCL_C_API CUresult cccl_device_reduce_build_ex(
|
||||
cccl_device_reduce_build_result_t* build,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
cccl_op_t op,
|
||||
cccl_value_t init,
|
||||
cccl_determinism_t determinism,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path,
|
||||
cccl_build_config* config);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_reduce(
|
||||
cccl_device_reduce_build_result_t build,
|
||||
void* d_temp_storage,
|
||||
size_t* temp_storage_bytes,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
uint64_t num_items,
|
||||
cccl_op_t op,
|
||||
cccl_value_t init,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_reduce_nondeterministic(
|
||||
cccl_device_reduce_build_result_t build,
|
||||
void* d_temp_storage,
|
||||
size_t* temp_storage_bytes,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
uint64_t num_items,
|
||||
cccl_op_t op,
|
||||
cccl_value_t init,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_reduce_cleanup(cccl_device_reduce_build_result_t* bld_ptr);
|
||||
|
||||
CCCL_C_EXTERN_C_END
|
||||
// NOLINTEND(modernize-use-using)
|
||||
127
cccl_upstream/c/parallel.v2/include/cccl/c/scan.h
Normal file
127
cccl_upstream/c/parallel.v2/include/cccl/c/scan.h
Normal file
@@ -0,0 +1,127 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA Core Compute Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
// NOLINTBEGIN(modernize-use-using)
|
||||
|
||||
#ifndef CCCL_C_EXPERIMENTAL
|
||||
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
|
||||
#endif // !CCCL_C_EXPERIMENTAL
|
||||
|
||||
#include <cuda.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <cccl/c/extern_c.h>
|
||||
#include <cccl/c/types.h>
|
||||
|
||||
CCCL_C_EXTERN_C_BEGIN
|
||||
|
||||
typedef struct cccl_device_scan_build_result_t
|
||||
{
|
||||
int cc;
|
||||
void* payload;
|
||||
size_t payload_size;
|
||||
void* jit_compiler;
|
||||
void* scan_fn;
|
||||
bool force_inclusive;
|
||||
cccl_init_kind_t init_kind;
|
||||
} cccl_device_scan_build_result_t;
|
||||
|
||||
CCCL_C_API CUresult cccl_device_scan_build(
|
||||
cccl_device_scan_build_result_t* build_ptr,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
cccl_op_t op,
|
||||
cccl_type_info init,
|
||||
bool force_inclusive,
|
||||
cccl_init_kind_t init_kind,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path);
|
||||
|
||||
// Extended version with build configuration
|
||||
CCCL_C_API CUresult cccl_device_scan_build_ex(
|
||||
cccl_device_scan_build_result_t* build_ptr,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
cccl_op_t op,
|
||||
cccl_type_info init,
|
||||
bool force_inclusive,
|
||||
cccl_init_kind_t init_kind,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path,
|
||||
cccl_build_config* config);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_exclusive_scan(
|
||||
cccl_device_scan_build_result_t build,
|
||||
void* d_temp_storage,
|
||||
size_t* temp_storage_bytes,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
uint64_t num_items,
|
||||
cccl_op_t op,
|
||||
cccl_value_t init,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_inclusive_scan(
|
||||
cccl_device_scan_build_result_t build,
|
||||
void* d_temp_storage,
|
||||
size_t* temp_storage_bytes,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
uint64_t num_items,
|
||||
cccl_op_t op,
|
||||
cccl_value_t init,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_exclusive_scan_future_value(
|
||||
cccl_device_scan_build_result_t build,
|
||||
void* d_temp_storage,
|
||||
size_t* temp_storage_bytes,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
uint64_t num_items,
|
||||
cccl_op_t op,
|
||||
cccl_iterator_t init,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_inclusive_scan_future_value(
|
||||
cccl_device_scan_build_result_t build,
|
||||
void* d_temp_storage,
|
||||
size_t* temp_storage_bytes,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
uint64_t num_items,
|
||||
cccl_op_t op,
|
||||
cccl_iterator_t init,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_inclusive_scan_no_init(
|
||||
cccl_device_scan_build_result_t build,
|
||||
void* d_temp_storage,
|
||||
size_t* temp_storage_bytes,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
uint64_t num_items,
|
||||
cccl_op_t op,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_scan_cleanup(cccl_device_scan_build_result_t* bld_ptr);
|
||||
|
||||
CCCL_C_EXTERN_C_END
|
||||
// NOLINTEND(modernize-use-using)
|
||||
@@ -0,0 +1,84 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA Core Compute Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
// NOLINTBEGIN(modernize-use-using)
|
||||
|
||||
#ifndef CCCL_C_EXPERIMENTAL
|
||||
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
|
||||
#endif // !CCCL_C_EXPERIMENTAL
|
||||
|
||||
#include <cuda.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <cccl/c/extern_c.h>
|
||||
#include <cccl/c/types.h>
|
||||
|
||||
CCCL_C_EXTERN_C_BEGIN
|
||||
|
||||
typedef struct cccl_device_segmented_reduce_build_result_t
|
||||
{
|
||||
int cc;
|
||||
void* payload;
|
||||
size_t payload_size;
|
||||
void* jit_compiler;
|
||||
void* segmented_reduce_fn;
|
||||
} cccl_device_segmented_reduce_build_result_t;
|
||||
|
||||
// TODO return a union of nvtx/cuda/nvrtc errors or a string?
|
||||
CCCL_C_API CUresult cccl_device_segmented_reduce_build(
|
||||
cccl_device_segmented_reduce_build_result_t* build,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
cccl_iterator_t begin_offset_in,
|
||||
cccl_iterator_t end_offset_in,
|
||||
cccl_op_t op,
|
||||
cccl_value_t init,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path);
|
||||
|
||||
// Extended version with build configuration
|
||||
CCCL_C_API CUresult cccl_device_segmented_reduce_build_ex(
|
||||
cccl_device_segmented_reduce_build_result_t* build,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
cccl_iterator_t begin_offset_in,
|
||||
cccl_iterator_t end_offset_in,
|
||||
cccl_op_t op,
|
||||
cccl_value_t init,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path,
|
||||
cccl_build_config* config);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_segmented_reduce(
|
||||
cccl_device_segmented_reduce_build_result_t build,
|
||||
void* d_temp_storage,
|
||||
size_t* temp_storage_bytes,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
uint64_t num_segments,
|
||||
cccl_iterator_t begin_offset_in,
|
||||
cccl_iterator_t end_offset_in,
|
||||
cccl_op_t op,
|
||||
cccl_value_t init,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_segmented_reduce_cleanup(cccl_device_segmented_reduce_build_result_t* bld_ptr);
|
||||
|
||||
CCCL_C_EXTERN_C_END
|
||||
// NOLINTEND(modernize-use-using)
|
||||
91
cccl_upstream/c/parallel.v2/include/cccl/c/segmented_sort.h
Normal file
91
cccl_upstream/c/parallel.v2/include/cccl/c/segmented_sort.h
Normal file
@@ -0,0 +1,91 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA Core Compute Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
// NOLINTBEGIN(modernize-use-using)
|
||||
|
||||
#ifndef CCCL_C_EXPERIMENTAL
|
||||
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
|
||||
#endif // !CCCL_C_EXPERIMENTAL
|
||||
|
||||
#include <cuda.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <cccl/c/extern_c.h>
|
||||
#include <cccl/c/types.h>
|
||||
|
||||
CCCL_C_EXTERN_C_BEGIN
|
||||
|
||||
typedef struct cccl_device_segmented_sort_build_result_t
|
||||
{
|
||||
int cc;
|
||||
void* payload;
|
||||
size_t payload_size;
|
||||
void* jit_compiler; /* Owns both wrappers below — one TU, one cubin */
|
||||
void* sort_fn; /* Wrapper around CUB's copy-overload (selector always 0) */
|
||||
void* sort_fn_overwrite; /* Wrapper around CUB's DoubleBuffer overload; reports selector */
|
||||
cccl_type_info key_type;
|
||||
cccl_type_info value_type;
|
||||
cccl_sort_order_t order;
|
||||
int keys_only; /* 1 if keys-only sort, 0 if key-value pairs */
|
||||
} cccl_device_segmented_sort_build_result_t;
|
||||
|
||||
// TODO return a union of nvtx/cuda/nvrtc errors or a string?
|
||||
CCCL_C_API CUresult cccl_device_segmented_sort_build(
|
||||
cccl_device_segmented_sort_build_result_t* build,
|
||||
cccl_sort_order_t sort_order,
|
||||
cccl_iterator_t d_keys_in,
|
||||
cccl_iterator_t d_values_in,
|
||||
cccl_iterator_t begin_offset_in,
|
||||
cccl_iterator_t end_offset_in,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path);
|
||||
|
||||
// Extended version with build configuration
|
||||
CCCL_C_API CUresult cccl_device_segmented_sort_build_ex(
|
||||
cccl_device_segmented_sort_build_result_t* build,
|
||||
cccl_sort_order_t sort_order,
|
||||
cccl_iterator_t d_keys_in,
|
||||
cccl_iterator_t d_values_in,
|
||||
cccl_iterator_t begin_offset_in,
|
||||
cccl_iterator_t end_offset_in,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path,
|
||||
cccl_build_config* config);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_segmented_sort(
|
||||
cccl_device_segmented_sort_build_result_t build,
|
||||
void* d_temp_storage,
|
||||
size_t* temp_storage_bytes,
|
||||
cccl_iterator_t d_keys_in,
|
||||
cccl_iterator_t d_keys_out,
|
||||
cccl_iterator_t d_values_in,
|
||||
cccl_iterator_t d_values_out,
|
||||
uint64_t num_items,
|
||||
uint64_t num_segments,
|
||||
cccl_iterator_t begin_offset_in,
|
||||
cccl_iterator_t end_offset_in,
|
||||
bool is_overwrite_okay,
|
||||
int* selector,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_segmented_sort_cleanup(cccl_device_segmented_sort_build_result_t* bld_ptr);
|
||||
|
||||
CCCL_C_EXTERN_C_END
|
||||
// NOLINTEND(modernize-use-using)
|
||||
@@ -0,0 +1,87 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA Core Compute Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
// NOLINTBEGIN(modernize-use-using)
|
||||
|
||||
#ifndef CCCL_C_EXPERIMENTAL
|
||||
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
|
||||
#endif // !CCCL_C_EXPERIMENTAL
|
||||
|
||||
#include <cuda.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <cccl/c/extern_c.h>
|
||||
#include <cccl/c/types.h>
|
||||
|
||||
CCCL_C_EXTERN_C_BEGIN
|
||||
|
||||
typedef struct cccl_device_three_way_partition_build_result_t
|
||||
{
|
||||
int cc;
|
||||
void* payload;
|
||||
size_t payload_size;
|
||||
void* jit_compiler;
|
||||
void* three_way_partition_fn;
|
||||
} cccl_device_three_way_partition_build_result_t;
|
||||
|
||||
// TODO return a union of nvtx/cuda/nvrtc errors or a string?
|
||||
CCCL_C_API CUresult cccl_device_three_way_partition_build(
|
||||
cccl_device_three_way_partition_build_result_t* build,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_first_part_out,
|
||||
cccl_iterator_t d_second_part_out,
|
||||
cccl_iterator_t d_unselected_out,
|
||||
cccl_iterator_t d_num_selected_out,
|
||||
cccl_op_t select_first_part_op,
|
||||
cccl_op_t select_second_part_op,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path);
|
||||
|
||||
// Extended version with build configuration
|
||||
CCCL_C_API CUresult cccl_device_three_way_partition_build_ex(
|
||||
cccl_device_three_way_partition_build_result_t* build,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_first_part_out,
|
||||
cccl_iterator_t d_second_part_out,
|
||||
cccl_iterator_t d_unselected_out,
|
||||
cccl_iterator_t d_num_selected_out,
|
||||
cccl_op_t select_first_part_op,
|
||||
cccl_op_t select_second_part_op,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path,
|
||||
cccl_build_config* config);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_three_way_partition(
|
||||
cccl_device_three_way_partition_build_result_t build,
|
||||
void* d_temp_storage,
|
||||
size_t* temp_storage_bytes,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_first_part_out,
|
||||
cccl_iterator_t d_second_part_out,
|
||||
cccl_iterator_t d_unselected_out,
|
||||
cccl_iterator_t d_num_selected_out,
|
||||
cccl_op_t select_first_part_op,
|
||||
cccl_op_t select_second_part_op,
|
||||
uint64_t num_items,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_three_way_partition_cleanup(cccl_device_three_way_partition_build_result_t* bld_ptr);
|
||||
|
||||
CCCL_C_EXTERN_C_END
|
||||
// NOLINTEND(modernize-use-using)
|
||||
113
cccl_upstream/c/parallel.v2/include/cccl/c/transform.h
Normal file
113
cccl_upstream/c/parallel.v2/include/cccl/c/transform.h
Normal file
@@ -0,0 +1,113 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA Core Compute Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
// NOLINTBEGIN(modernize-use-using)
|
||||
|
||||
#ifndef CCCL_C_EXPERIMENTAL
|
||||
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
|
||||
#endif // !CCCL_C_EXPERIMENTAL
|
||||
|
||||
#include <cuda.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <cccl/c/extern_c.h>
|
||||
#include <cccl/c/types.h>
|
||||
|
||||
CCCL_C_EXTERN_C_BEGIN
|
||||
|
||||
typedef struct cccl_device_transform_build_result_t
|
||||
{
|
||||
int cc;
|
||||
void* payload;
|
||||
size_t payload_size;
|
||||
void* jit_compiler;
|
||||
#if defined(_WIN32)
|
||||
// Opaque state for serializing CUB's lazy first-call initialization.
|
||||
void* first_call_state;
|
||||
#endif // _WIN32
|
||||
void* transform_fn;
|
||||
} cccl_device_transform_build_result_t;
|
||||
|
||||
CCCL_C_API CUresult cccl_device_unary_transform_build(
|
||||
cccl_device_transform_build_result_t* build_ptr,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
cccl_op_t op,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path);
|
||||
|
||||
// Extended version with build configuration
|
||||
CCCL_C_API CUresult cccl_device_unary_transform_build_ex(
|
||||
cccl_device_transform_build_result_t* build_ptr,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
cccl_op_t op,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path,
|
||||
cccl_build_config* config);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_unary_transform(
|
||||
cccl_device_transform_build_result_t build,
|
||||
cccl_iterator_t d_in,
|
||||
cccl_iterator_t d_out,
|
||||
uint64_t num_items,
|
||||
cccl_op_t op,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_binary_transform_build(
|
||||
cccl_device_transform_build_result_t* build_ptr,
|
||||
cccl_iterator_t d_in1,
|
||||
cccl_iterator_t d_in2,
|
||||
cccl_iterator_t d_out,
|
||||
cccl_op_t op,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path);
|
||||
|
||||
// Extended version with build configuration
|
||||
CCCL_C_API CUresult cccl_device_binary_transform_build_ex(
|
||||
cccl_device_transform_build_result_t* build_ptr,
|
||||
cccl_iterator_t d_in1,
|
||||
cccl_iterator_t d_in2,
|
||||
cccl_iterator_t d_out,
|
||||
cccl_op_t op,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path,
|
||||
cccl_build_config* config);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_binary_transform(
|
||||
cccl_device_transform_build_result_t build,
|
||||
cccl_iterator_t d_in1,
|
||||
cccl_iterator_t d_in2,
|
||||
cccl_iterator_t d_out,
|
||||
uint64_t num_items,
|
||||
cccl_op_t op,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_transform_cleanup(cccl_device_transform_build_result_t* bld_ptr);
|
||||
|
||||
CCCL_C_EXTERN_C_END
|
||||
// NOLINTEND(modernize-use-using)
|
||||
188
cccl_upstream/c/parallel.v2/include/cccl/c/types.h
Normal file
188
cccl_upstream/c/parallel.v2/include/cccl/c/types.h
Normal file
@@ -0,0 +1,188 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA Core Compute Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
// NOLINTBEGIN(modernize-use-using)
|
||||
|
||||
#ifndef CCCL_C_EXPERIMENTAL
|
||||
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
|
||||
#endif // !CCCL_C_EXPERIMENTAL
|
||||
|
||||
#if defined(_WIN32)
|
||||
# define CCCL_C_API __declspec(dllexport)
|
||||
#else // ^^^ _WIN32 ^^^ / vvv !_WIN32 vvv
|
||||
# define CCCL_C_API __attribute__((__visibility__("default")))
|
||||
#endif // !_WIN32
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <cccl/c/extern_c.h>
|
||||
|
||||
CCCL_C_EXTERN_C_BEGIN
|
||||
|
||||
typedef enum cccl_type_enum
|
||||
{
|
||||
CCCL_INT8 = 0,
|
||||
CCCL_INT16 = 1,
|
||||
CCCL_INT32 = 2,
|
||||
CCCL_INT64 = 3,
|
||||
CCCL_UINT8 = 4,
|
||||
CCCL_UINT16 = 5,
|
||||
CCCL_UINT32 = 6,
|
||||
CCCL_UINT64 = 7,
|
||||
CCCL_FLOAT16 = 8, // This may be unsupported if _CCCL_HAS_NVFP16() is false but we can't include the header to check
|
||||
// that here
|
||||
CCCL_FLOAT32 = 9,
|
||||
CCCL_FLOAT64 = 10,
|
||||
CCCL_STORAGE = 11,
|
||||
CCCL_BOOLEAN = 12,
|
||||
} cccl_type_enum;
|
||||
|
||||
typedef struct cccl_type_info
|
||||
{
|
||||
size_t size;
|
||||
size_t alignment;
|
||||
cccl_type_enum type;
|
||||
} cccl_type_info;
|
||||
|
||||
typedef enum cccl_op_kind_t
|
||||
{
|
||||
// Arbitrary semantics, without state.
|
||||
CCCL_STATELESS = 0,
|
||||
// Arbitrary semantics, with state.
|
||||
CCCL_STATEFUL = 1,
|
||||
// Well-known semantics, required to be stateless.
|
||||
// Equivalent to corresponding function objects in C++'s <functional>.
|
||||
// If the types involved are primitive, only the kind field is necessary.
|
||||
// Otherwise, the cccl_op_t object must also contain the rest of the fields,
|
||||
// as appropriate.
|
||||
CCCL_PLUS = 2,
|
||||
CCCL_MINUS = 3,
|
||||
CCCL_MULTIPLIES = 4,
|
||||
CCCL_DIVIDES = 5,
|
||||
CCCL_MODULUS = 6,
|
||||
CCCL_EQUAL_TO = 7,
|
||||
CCCL_NOT_EQUAL_TO = 8,
|
||||
CCCL_GREATER = 9,
|
||||
CCCL_LESS = 10,
|
||||
CCCL_GREATER_EQUAL = 11,
|
||||
CCCL_LESS_EQUAL = 12,
|
||||
CCCL_LOGICAL_AND = 13,
|
||||
CCCL_LOGICAL_OR = 14,
|
||||
CCCL_LOGICAL_NOT = 15,
|
||||
CCCL_BIT_AND = 16,
|
||||
CCCL_BIT_OR = 17,
|
||||
CCCL_BIT_XOR = 18,
|
||||
CCCL_BIT_NOT = 19,
|
||||
CCCL_IDENTITY = 20,
|
||||
CCCL_NEGATE = 21,
|
||||
CCCL_MINIMUM = 22,
|
||||
CCCL_MAXIMUM = 23,
|
||||
} cccl_op_kind_t;
|
||||
|
||||
typedef enum cccl_op_code_type
|
||||
{
|
||||
CCCL_OP_LTOIR = 0, // Pre-compiled LTO-IR (escape hatch for callers with existing nvcc -dlto artifacts).
|
||||
// LTO-IR is a binary container passed to nvJitLink at the PTX level — the LLVM optimizer
|
||||
// never sees it, so the operator cannot be inlined into the CUB kernel and pays a real
|
||||
// CALL on every iteration. CCCL_OP_LLVM_IR feeds LLVM's bitcode linker instead, which
|
||||
// merges the operator into the CUB module before PTX codegen and enables full inlining.
|
||||
// Prefer CCCL_OP_LLVM_IR or CCCL_OP_CPP_SOURCE for any new code.
|
||||
CCCL_OP_CPP_SOURCE = 1, // C++ source code (compiled to LLVM bitcode by hostjit's Clang).
|
||||
CCCL_OP_LLVM_IR = 2 // LLVM bitcode (recommended) — merges into the CUB module before PTX gen, so inlines.
|
||||
} cccl_op_code_type;
|
||||
|
||||
typedef struct cccl_op_t
|
||||
{
|
||||
cccl_op_kind_t type;
|
||||
const char* name;
|
||||
const char* code;
|
||||
size_t code_size;
|
||||
cccl_op_code_type code_type;
|
||||
size_t size;
|
||||
size_t alignment;
|
||||
void* state;
|
||||
const char** extra_ltoirs;
|
||||
size_t* extra_ltoir_sizes;
|
||||
size_t num_extra_ltoirs;
|
||||
cccl_op_code_type* extra_code_types;
|
||||
} cccl_op_t;
|
||||
|
||||
typedef struct cccl_build_config
|
||||
{
|
||||
const char** extra_compile_flags; // e.g., {"-DENABLE_FAST_MATH", "-O3"}
|
||||
size_t num_extra_compile_flags;
|
||||
const char** extra_include_dirs; // e.g., {"/path/to/my/headers"}
|
||||
size_t num_extra_include_dirs;
|
||||
int enable_pch; // Cache precompiled headers on disk to speed up repeated builds
|
||||
int verbose; // Log PCH generation/usage and compiler args to build diagnostics
|
||||
} cccl_build_config;
|
||||
|
||||
typedef enum cccl_iterator_kind_t
|
||||
{
|
||||
CCCL_POINTER = 0,
|
||||
CCCL_ITERATOR = 1,
|
||||
} cccl_iterator_kind_t;
|
||||
|
||||
typedef struct cccl_value_t
|
||||
{
|
||||
cccl_type_info type;
|
||||
void* state;
|
||||
} cccl_value_t;
|
||||
|
||||
typedef union
|
||||
{
|
||||
int64_t signed_offset;
|
||||
uint64_t unsigned_offset;
|
||||
} cccl_increment_t;
|
||||
|
||||
typedef void (*cccl_host_op_fn_ptr_t)(void*, cccl_increment_t);
|
||||
|
||||
typedef struct cccl_iterator_t
|
||||
{
|
||||
size_t size;
|
||||
size_t alignment;
|
||||
cccl_iterator_kind_t type;
|
||||
cccl_op_t advance;
|
||||
cccl_op_t dereference;
|
||||
cccl_type_info value_type;
|
||||
void* state;
|
||||
cccl_host_op_fn_ptr_t host_advance;
|
||||
} cccl_iterator_t;
|
||||
|
||||
typedef enum cccl_sort_order_t
|
||||
{
|
||||
CCCL_ASCENDING = 0,
|
||||
CCCL_DESCENDING = 1,
|
||||
} cccl_sort_order_t;
|
||||
|
||||
typedef enum cccl_init_kind_t
|
||||
{
|
||||
CCCL_VALUE_INIT = 0,
|
||||
CCCL_FUTURE_VALUE_INIT = 1,
|
||||
CCCL_NO_INIT = 2,
|
||||
} cccl_init_kind_t;
|
||||
|
||||
typedef enum cccl_determinism_t
|
||||
{
|
||||
CCCL_NOT_GUARANTEED = 0,
|
||||
CCCL_RUN_TO_RUN = 1,
|
||||
CCCL_GPU_TO_GPU = 2,
|
||||
} cccl_determinism_t;
|
||||
|
||||
typedef enum cccl_binary_search_mode_t
|
||||
{
|
||||
CCCL_BINARY_SEARCH_LOWER_BOUND = 0,
|
||||
CCCL_BINARY_SEARCH_UPPER_BOUND = 1,
|
||||
} cccl_binary_search_mode_t;
|
||||
|
||||
CCCL_C_EXTERN_C_END
|
||||
// NOLINTEND(modernize-use-using)
|
||||
83
cccl_upstream/c/parallel.v2/include/cccl/c/unique_by_key.h
Normal file
83
cccl_upstream/c/parallel.v2/include/cccl/c/unique_by_key.h
Normal file
@@ -0,0 +1,83 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA Core Compute Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
// NOLINTBEGIN(modernize-use-using)
|
||||
|
||||
#ifndef CCCL_C_EXPERIMENTAL
|
||||
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
|
||||
#endif // !CCCL_C_EXPERIMENTAL
|
||||
|
||||
#include <cuda.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <cccl/c/extern_c.h>
|
||||
#include <cccl/c/types.h>
|
||||
|
||||
CCCL_C_EXTERN_C_BEGIN
|
||||
|
||||
typedef struct cccl_device_unique_by_key_build_result_t
|
||||
{
|
||||
int cc;
|
||||
void* payload;
|
||||
size_t payload_size;
|
||||
void* jit_compiler;
|
||||
void* unique_by_key_fn;
|
||||
} cccl_device_unique_by_key_build_result_t;
|
||||
|
||||
CCCL_C_API CUresult cccl_device_unique_by_key_build(
|
||||
cccl_device_unique_by_key_build_result_t* build,
|
||||
cccl_iterator_t d_keys_in,
|
||||
cccl_iterator_t d_values_in,
|
||||
cccl_iterator_t d_keys_out,
|
||||
cccl_iterator_t d_values_out,
|
||||
cccl_iterator_t d_num_selected_out,
|
||||
cccl_op_t op,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path);
|
||||
|
||||
// Extended version with build configuration
|
||||
CCCL_C_API CUresult cccl_device_unique_by_key_build_ex(
|
||||
cccl_device_unique_by_key_build_result_t* build,
|
||||
cccl_iterator_t d_keys_in,
|
||||
cccl_iterator_t d_values_in,
|
||||
cccl_iterator_t d_keys_out,
|
||||
cccl_iterator_t d_values_out,
|
||||
cccl_iterator_t d_num_selected_out,
|
||||
cccl_op_t op,
|
||||
int cc_major,
|
||||
int cc_minor,
|
||||
const char* cub_path,
|
||||
const char* thrust_path,
|
||||
const char* libcudacxx_path,
|
||||
const char* ctk_path,
|
||||
cccl_build_config* config);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_unique_by_key(
|
||||
cccl_device_unique_by_key_build_result_t build,
|
||||
void* d_temp_storage,
|
||||
size_t* temp_storage_bytes,
|
||||
cccl_iterator_t d_keys_in,
|
||||
cccl_iterator_t d_values_in,
|
||||
cccl_iterator_t d_keys_out,
|
||||
cccl_iterator_t d_values_out,
|
||||
cccl_iterator_t d_num_selected_out,
|
||||
cccl_op_t op,
|
||||
uint64_t num_items,
|
||||
CUstream stream);
|
||||
|
||||
CCCL_C_API CUresult cccl_device_unique_by_key_cleanup(cccl_device_unique_by_key_build_result_t* bld_ptr);
|
||||
|
||||
CCCL_C_EXTERN_C_END
|
||||
// NOLINTEND(modernize-use-using)
|
||||
Reference in New Issue
Block a user