[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0
This commit is contained in:
EngineX CI
2026-07-30 09:35:51 +00:00
parent b4d01f481e
commit 56fd68e7dd
8871 changed files with 1454674 additions and 0 deletions

View File

@@ -0,0 +1,82 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_binary_search_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler; // hostjit::JITCompiler*
#if defined(_WIN32)
// Opaque state for serializing CUB's lazy first-call initialization.
void* first_call_state;
#endif // _WIN32
void* binary_search_fn; // int(*)(void*, ull, void*, ull, void*, void*, void*)
} cccl_device_binary_search_build_result_t;
CCCL_C_API CUresult cccl_device_binary_search_build(
cccl_device_binary_search_build_result_t* build,
cccl_binary_search_mode_t mode,
cccl_iterator_t d_data,
cccl_iterator_t d_values,
cccl_iterator_t d_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_binary_search_build_ex(
cccl_device_binary_search_build_result_t* build,
cccl_binary_search_mode_t mode,
cccl_iterator_t d_data,
cccl_iterator_t d_values,
cccl_iterator_t d_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_binary_search(
cccl_device_binary_search_build_result_t build,
cccl_iterator_t d_data,
uint64_t num_items,
cccl_iterator_t d_values,
uint64_t num_values,
cccl_iterator_t d_out,
cccl_op_t op,
CUstream stream);
CCCL_C_API CUresult cccl_device_binary_search_cleanup(cccl_device_binary_search_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

View File

@@ -0,0 +1,23 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
#ifdef __cplusplus
# define CCCL_C_EXTERN_C_BEGIN extern "C" {
# define CCCL_C_EXTERN_C_END }
#else
# define CCCL_C_EXTERN_C_BEGIN
# define CCCL_C_EXTERN_C_END
#endif

View File

@@ -0,0 +1,65 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_for_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler; // hostjit::JITCompiler*
void* for_fn; // int(*)(void*, unsigned long long, void*)
} cccl_device_for_build_result_t;
CCCL_C_API CUresult cccl_device_for_build(
cccl_device_for_build_result_t* build,
cccl_iterator_t d_data,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_for_build_ex(
cccl_device_for_build_result_t* build,
cccl_iterator_t d_data,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_for(
cccl_device_for_build_result_t build, cccl_iterator_t d_data, uint64_t num_items, cccl_op_t op, CUstream stream);
CCCL_C_API CUresult cccl_device_for_cleanup(cccl_device_for_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

View File

@@ -0,0 +1,96 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdbool.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_histogram_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler;
void* histogram_fn;
cccl_type_info counter_type;
cccl_type_info level_type;
cccl_type_info sample_type;
int num_channels;
int num_active_channels;
} cccl_device_histogram_build_result_t;
CCCL_C_API CUresult cccl_device_histogram_build(
cccl_device_histogram_build_result_t* build,
int num_channels,
int num_active_channels,
cccl_iterator_t d_samples,
int num_output_levels_val,
cccl_iterator_t d_output_histograms,
cccl_type_info level_type,
int64_t num_rows,
int64_t row_stride_samples,
bool is_evenly_segmented,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_histogram_build_ex(
cccl_device_histogram_build_result_t* build,
int num_channels,
int num_active_channels,
cccl_iterator_t d_samples,
int num_output_levels_val,
cccl_iterator_t d_output_histograms,
cccl_type_info level_type,
int64_t num_rows,
int64_t row_stride_samples,
bool is_evenly_segmented,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_histogram_even(
cccl_device_histogram_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_samples,
cccl_iterator_t d_output_histograms,
cccl_value_t num_output_levels,
cccl_value_t lower_level,
cccl_value_t upper_level,
int64_t num_row_pixels,
int64_t num_rows,
int64_t row_stride_samples,
CUstream stream);
CCCL_C_API CUresult cccl_device_histogram_cleanup(cccl_device_histogram_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

View File

@@ -0,0 +1,86 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_merge_sort_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler;
void* sort_fn;
// 1 if the build compiled SortKeysCopy (no items), 0 if SortPairsCopy. The
// run function dispatches on this so the value-vs-pairs decision doesn't
// have to be re-derived from the iterator arguments.
int keys_only;
cccl_type_info key_type;
cccl_type_info item_type;
} cccl_device_merge_sort_build_result_t;
CCCL_C_API CUresult cccl_device_merge_sort_build(
cccl_device_merge_sort_build_result_t* build,
cccl_iterator_t d_in_keys,
cccl_iterator_t d_in_items,
cccl_iterator_t d_out_keys,
cccl_iterator_t d_out_items,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_merge_sort_build_ex(
cccl_device_merge_sort_build_result_t* build,
cccl_iterator_t d_in_keys,
cccl_iterator_t d_in_items,
cccl_iterator_t d_out_keys,
cccl_iterator_t d_out_items,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_merge_sort(
cccl_device_merge_sort_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in_keys,
cccl_iterator_t d_in_items,
cccl_iterator_t d_out_keys,
cccl_iterator_t d_out_items,
uint64_t num_items,
cccl_op_t op,
CUstream stream);
CCCL_C_API CUresult cccl_device_merge_sort_cleanup(cccl_device_merge_sort_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

View File

@@ -0,0 +1,90 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdbool.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_radix_sort_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler; /* Owns both wrappers below — one TU, one cubin */
void* sort_fn; /* Wrapper around CUB's copy-overload (selector always 0) */
void* sort_fn_overwrite; /* Wrapper around CUB's DoubleBuffer overload; reports selector */
cccl_type_info key_type;
cccl_type_info value_type;
cccl_sort_order_t order;
int keys_only; /* 1 if keys-only sort, 0 if key-value pairs */
} cccl_device_radix_sort_build_result_t;
CCCL_C_API CUresult cccl_device_radix_sort_build(
cccl_device_radix_sort_build_result_t* build,
cccl_sort_order_t sort_order,
cccl_iterator_t input_keys_it,
cccl_iterator_t input_values_it,
cccl_op_t decomposer,
const char* decomposer_return_type,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_radix_sort_build_ex(
cccl_device_radix_sort_build_result_t* build,
cccl_sort_order_t sort_order,
cccl_iterator_t input_keys_it,
cccl_iterator_t input_values_it,
cccl_op_t decomposer,
const char* decomposer_return_type,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_radix_sort(
cccl_device_radix_sort_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_keys_out,
cccl_iterator_t d_values_in,
cccl_iterator_t d_values_out,
cccl_op_t decomposer,
uint64_t num_items,
int begin_bit,
int end_bit,
bool is_overwrite_okay,
int* selector,
CUstream stream);
CCCL_C_API CUresult cccl_device_radix_sort_cleanup(cccl_device_radix_sort_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

View File

@@ -0,0 +1,94 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_reduce_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler; // hostjit::JITCompiler*
void* reduce_fn; // int(*)(void*, size_t*, void*, void*, unsigned long long, void*, void*, void*) — trailing void* is
// the CUstream
uint64_t accumulator_size;
cccl_determinism_t determinism;
} cccl_device_reduce_build_result_t;
// TODO return a union of nvtx/cuda/nvrtc errors or a string?
CCCL_C_API CUresult cccl_device_reduce_build(
cccl_device_reduce_build_result_t* build,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_op_t op,
cccl_value_t init,
cccl_determinism_t determinism,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_reduce_build_ex(
cccl_device_reduce_build_result_t* build,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_op_t op,
cccl_value_t init,
cccl_determinism_t determinism,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_reduce(
cccl_device_reduce_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
cccl_value_t init,
CUstream stream);
CCCL_C_API CUresult cccl_device_reduce_nondeterministic(
cccl_device_reduce_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
cccl_value_t init,
CUstream stream);
CCCL_C_API CUresult cccl_device_reduce_cleanup(cccl_device_reduce_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

View File

@@ -0,0 +1,127 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdbool.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_scan_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler;
void* scan_fn;
bool force_inclusive;
cccl_init_kind_t init_kind;
} cccl_device_scan_build_result_t;
CCCL_C_API CUresult cccl_device_scan_build(
cccl_device_scan_build_result_t* build_ptr,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_op_t op,
cccl_type_info init,
bool force_inclusive,
cccl_init_kind_t init_kind,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_scan_build_ex(
cccl_device_scan_build_result_t* build_ptr,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_op_t op,
cccl_type_info init,
bool force_inclusive,
cccl_init_kind_t init_kind,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_exclusive_scan(
cccl_device_scan_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
cccl_value_t init,
CUstream stream);
CCCL_C_API CUresult cccl_device_inclusive_scan(
cccl_device_scan_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
cccl_value_t init,
CUstream stream);
CCCL_C_API CUresult cccl_device_exclusive_scan_future_value(
cccl_device_scan_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
cccl_iterator_t init,
CUstream stream);
CCCL_C_API CUresult cccl_device_inclusive_scan_future_value(
cccl_device_scan_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
cccl_iterator_t init,
CUstream stream);
CCCL_C_API CUresult cccl_device_inclusive_scan_no_init(
cccl_device_scan_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
CUstream stream);
CCCL_C_API CUresult cccl_device_scan_cleanup(cccl_device_scan_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

View File

@@ -0,0 +1,84 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_segmented_reduce_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler;
void* segmented_reduce_fn;
} cccl_device_segmented_reduce_build_result_t;
// TODO return a union of nvtx/cuda/nvrtc errors or a string?
CCCL_C_API CUresult cccl_device_segmented_reduce_build(
cccl_device_segmented_reduce_build_result_t* build,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_iterator_t begin_offset_in,
cccl_iterator_t end_offset_in,
cccl_op_t op,
cccl_value_t init,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_segmented_reduce_build_ex(
cccl_device_segmented_reduce_build_result_t* build,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_iterator_t begin_offset_in,
cccl_iterator_t end_offset_in,
cccl_op_t op,
cccl_value_t init,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_segmented_reduce(
cccl_device_segmented_reduce_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_segments,
cccl_iterator_t begin_offset_in,
cccl_iterator_t end_offset_in,
cccl_op_t op,
cccl_value_t init,
CUstream stream);
CCCL_C_API CUresult cccl_device_segmented_reduce_cleanup(cccl_device_segmented_reduce_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

View File

@@ -0,0 +1,91 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdbool.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_segmented_sort_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler; /* Owns both wrappers below — one TU, one cubin */
void* sort_fn; /* Wrapper around CUB's copy-overload (selector always 0) */
void* sort_fn_overwrite; /* Wrapper around CUB's DoubleBuffer overload; reports selector */
cccl_type_info key_type;
cccl_type_info value_type;
cccl_sort_order_t order;
int keys_only; /* 1 if keys-only sort, 0 if key-value pairs */
} cccl_device_segmented_sort_build_result_t;
// TODO return a union of nvtx/cuda/nvrtc errors or a string?
CCCL_C_API CUresult cccl_device_segmented_sort_build(
cccl_device_segmented_sort_build_result_t* build,
cccl_sort_order_t sort_order,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_values_in,
cccl_iterator_t begin_offset_in,
cccl_iterator_t end_offset_in,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_segmented_sort_build_ex(
cccl_device_segmented_sort_build_result_t* build,
cccl_sort_order_t sort_order,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_values_in,
cccl_iterator_t begin_offset_in,
cccl_iterator_t end_offset_in,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_segmented_sort(
cccl_device_segmented_sort_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_keys_out,
cccl_iterator_t d_values_in,
cccl_iterator_t d_values_out,
uint64_t num_items,
uint64_t num_segments,
cccl_iterator_t begin_offset_in,
cccl_iterator_t end_offset_in,
bool is_overwrite_okay,
int* selector,
CUstream stream);
CCCL_C_API CUresult cccl_device_segmented_sort_cleanup(cccl_device_segmented_sort_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

View File

@@ -0,0 +1,87 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_three_way_partition_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler;
void* three_way_partition_fn;
} cccl_device_three_way_partition_build_result_t;
// TODO return a union of nvtx/cuda/nvrtc errors or a string?
CCCL_C_API CUresult cccl_device_three_way_partition_build(
cccl_device_three_way_partition_build_result_t* build,
cccl_iterator_t d_in,
cccl_iterator_t d_first_part_out,
cccl_iterator_t d_second_part_out,
cccl_iterator_t d_unselected_out,
cccl_iterator_t d_num_selected_out,
cccl_op_t select_first_part_op,
cccl_op_t select_second_part_op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_three_way_partition_build_ex(
cccl_device_three_way_partition_build_result_t* build,
cccl_iterator_t d_in,
cccl_iterator_t d_first_part_out,
cccl_iterator_t d_second_part_out,
cccl_iterator_t d_unselected_out,
cccl_iterator_t d_num_selected_out,
cccl_op_t select_first_part_op,
cccl_op_t select_second_part_op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_three_way_partition(
cccl_device_three_way_partition_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_in,
cccl_iterator_t d_first_part_out,
cccl_iterator_t d_second_part_out,
cccl_iterator_t d_unselected_out,
cccl_iterator_t d_num_selected_out,
cccl_op_t select_first_part_op,
cccl_op_t select_second_part_op,
uint64_t num_items,
CUstream stream);
CCCL_C_API CUresult cccl_device_three_way_partition_cleanup(cccl_device_three_way_partition_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

View File

@@ -0,0 +1,113 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_transform_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler;
#if defined(_WIN32)
// Opaque state for serializing CUB's lazy first-call initialization.
void* first_call_state;
#endif // _WIN32
void* transform_fn;
} cccl_device_transform_build_result_t;
CCCL_C_API CUresult cccl_device_unary_transform_build(
cccl_device_transform_build_result_t* build_ptr,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_unary_transform_build_ex(
cccl_device_transform_build_result_t* build_ptr,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_unary_transform(
cccl_device_transform_build_result_t build,
cccl_iterator_t d_in,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
CUstream stream);
CCCL_C_API CUresult cccl_device_binary_transform_build(
cccl_device_transform_build_result_t* build_ptr,
cccl_iterator_t d_in1,
cccl_iterator_t d_in2,
cccl_iterator_t d_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_binary_transform_build_ex(
cccl_device_transform_build_result_t* build_ptr,
cccl_iterator_t d_in1,
cccl_iterator_t d_in2,
cccl_iterator_t d_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_binary_transform(
cccl_device_transform_build_result_t build,
cccl_iterator_t d_in1,
cccl_iterator_t d_in2,
cccl_iterator_t d_out,
uint64_t num_items,
cccl_op_t op,
CUstream stream);
CCCL_C_API CUresult cccl_device_transform_cleanup(cccl_device_transform_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

View File

@@ -0,0 +1,188 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#if defined(_WIN32)
# define CCCL_C_API __declspec(dllexport)
#else // ^^^ _WIN32 ^^^ / vvv !_WIN32 vvv
# define CCCL_C_API __attribute__((__visibility__("default")))
#endif // !_WIN32
#include <stddef.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
CCCL_C_EXTERN_C_BEGIN
typedef enum cccl_type_enum
{
CCCL_INT8 = 0,
CCCL_INT16 = 1,
CCCL_INT32 = 2,
CCCL_INT64 = 3,
CCCL_UINT8 = 4,
CCCL_UINT16 = 5,
CCCL_UINT32 = 6,
CCCL_UINT64 = 7,
CCCL_FLOAT16 = 8, // This may be unsupported if _CCCL_HAS_NVFP16() is false but we can't include the header to check
// that here
CCCL_FLOAT32 = 9,
CCCL_FLOAT64 = 10,
CCCL_STORAGE = 11,
CCCL_BOOLEAN = 12,
} cccl_type_enum;
typedef struct cccl_type_info
{
size_t size;
size_t alignment;
cccl_type_enum type;
} cccl_type_info;
typedef enum cccl_op_kind_t
{
// Arbitrary semantics, without state.
CCCL_STATELESS = 0,
// Arbitrary semantics, with state.
CCCL_STATEFUL = 1,
// Well-known semantics, required to be stateless.
// Equivalent to corresponding function objects in C++'s <functional>.
// If the types involved are primitive, only the kind field is necessary.
// Otherwise, the cccl_op_t object must also contain the rest of the fields,
// as appropriate.
CCCL_PLUS = 2,
CCCL_MINUS = 3,
CCCL_MULTIPLIES = 4,
CCCL_DIVIDES = 5,
CCCL_MODULUS = 6,
CCCL_EQUAL_TO = 7,
CCCL_NOT_EQUAL_TO = 8,
CCCL_GREATER = 9,
CCCL_LESS = 10,
CCCL_GREATER_EQUAL = 11,
CCCL_LESS_EQUAL = 12,
CCCL_LOGICAL_AND = 13,
CCCL_LOGICAL_OR = 14,
CCCL_LOGICAL_NOT = 15,
CCCL_BIT_AND = 16,
CCCL_BIT_OR = 17,
CCCL_BIT_XOR = 18,
CCCL_BIT_NOT = 19,
CCCL_IDENTITY = 20,
CCCL_NEGATE = 21,
CCCL_MINIMUM = 22,
CCCL_MAXIMUM = 23,
} cccl_op_kind_t;
typedef enum cccl_op_code_type
{
CCCL_OP_LTOIR = 0, // Pre-compiled LTO-IR (escape hatch for callers with existing nvcc -dlto artifacts).
// LTO-IR is a binary container passed to nvJitLink at the PTX level — the LLVM optimizer
// never sees it, so the operator cannot be inlined into the CUB kernel and pays a real
// CALL on every iteration. CCCL_OP_LLVM_IR feeds LLVM's bitcode linker instead, which
// merges the operator into the CUB module before PTX codegen and enables full inlining.
// Prefer CCCL_OP_LLVM_IR or CCCL_OP_CPP_SOURCE for any new code.
CCCL_OP_CPP_SOURCE = 1, // C++ source code (compiled to LLVM bitcode by hostjit's Clang).
CCCL_OP_LLVM_IR = 2 // LLVM bitcode (recommended) — merges into the CUB module before PTX gen, so inlines.
} cccl_op_code_type;
typedef struct cccl_op_t
{
cccl_op_kind_t type;
const char* name;
const char* code;
size_t code_size;
cccl_op_code_type code_type;
size_t size;
size_t alignment;
void* state;
const char** extra_ltoirs;
size_t* extra_ltoir_sizes;
size_t num_extra_ltoirs;
cccl_op_code_type* extra_code_types;
} cccl_op_t;
typedef struct cccl_build_config
{
const char** extra_compile_flags; // e.g., {"-DENABLE_FAST_MATH", "-O3"}
size_t num_extra_compile_flags;
const char** extra_include_dirs; // e.g., {"/path/to/my/headers"}
size_t num_extra_include_dirs;
int enable_pch; // Cache precompiled headers on disk to speed up repeated builds
int verbose; // Log PCH generation/usage and compiler args to build diagnostics
} cccl_build_config;
typedef enum cccl_iterator_kind_t
{
CCCL_POINTER = 0,
CCCL_ITERATOR = 1,
} cccl_iterator_kind_t;
typedef struct cccl_value_t
{
cccl_type_info type;
void* state;
} cccl_value_t;
typedef union
{
int64_t signed_offset;
uint64_t unsigned_offset;
} cccl_increment_t;
typedef void (*cccl_host_op_fn_ptr_t)(void*, cccl_increment_t);
typedef struct cccl_iterator_t
{
size_t size;
size_t alignment;
cccl_iterator_kind_t type;
cccl_op_t advance;
cccl_op_t dereference;
cccl_type_info value_type;
void* state;
cccl_host_op_fn_ptr_t host_advance;
} cccl_iterator_t;
typedef enum cccl_sort_order_t
{
CCCL_ASCENDING = 0,
CCCL_DESCENDING = 1,
} cccl_sort_order_t;
typedef enum cccl_init_kind_t
{
CCCL_VALUE_INIT = 0,
CCCL_FUTURE_VALUE_INIT = 1,
CCCL_NO_INIT = 2,
} cccl_init_kind_t;
typedef enum cccl_determinism_t
{
CCCL_NOT_GUARANTEED = 0,
CCCL_RUN_TO_RUN = 1,
CCCL_GPU_TO_GPU = 2,
} cccl_determinism_t;
typedef enum cccl_binary_search_mode_t
{
CCCL_BINARY_SEARCH_LOWER_BOUND = 0,
CCCL_BINARY_SEARCH_UPPER_BOUND = 1,
} cccl_binary_search_mode_t;
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)

View File

@@ -0,0 +1,83 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA Core Compute Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION.
//
//===----------------------------------------------------------------------===//
#pragma once
// NOLINTBEGIN(modernize-use-using)
#ifndef CCCL_C_EXPERIMENTAL
# error "C exposure is experimental and subject to change. Define CCCL_C_EXPERIMENTAL to acknowledge this notice."
#endif // !CCCL_C_EXPERIMENTAL
#include <cuda.h>
#include <stdint.h>
#include <cccl/c/extern_c.h>
#include <cccl/c/types.h>
CCCL_C_EXTERN_C_BEGIN
typedef struct cccl_device_unique_by_key_build_result_t
{
int cc;
void* payload;
size_t payload_size;
void* jit_compiler;
void* unique_by_key_fn;
} cccl_device_unique_by_key_build_result_t;
CCCL_C_API CUresult cccl_device_unique_by_key_build(
cccl_device_unique_by_key_build_result_t* build,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_values_in,
cccl_iterator_t d_keys_out,
cccl_iterator_t d_values_out,
cccl_iterator_t d_num_selected_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path);
// Extended version with build configuration
CCCL_C_API CUresult cccl_device_unique_by_key_build_ex(
cccl_device_unique_by_key_build_result_t* build,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_values_in,
cccl_iterator_t d_keys_out,
cccl_iterator_t d_values_out,
cccl_iterator_t d_num_selected_out,
cccl_op_t op,
int cc_major,
int cc_minor,
const char* cub_path,
const char* thrust_path,
const char* libcudacxx_path,
const char* ctk_path,
cccl_build_config* config);
CCCL_C_API CUresult cccl_device_unique_by_key(
cccl_device_unique_by_key_build_result_t build,
void* d_temp_storage,
size_t* temp_storage_bytes,
cccl_iterator_t d_keys_in,
cccl_iterator_t d_values_in,
cccl_iterator_t d_keys_out,
cccl_iterator_t d_values_out,
cccl_iterator_t d_num_selected_out,
cccl_op_t op,
uint64_t num_items,
CUstream stream);
CCCL_C_API CUresult cccl_device_unique_by_key_cleanup(cccl_device_unique_by_key_build_result_t* bld_ptr);
CCCL_C_EXTERN_C_END
// NOLINTEND(modernize-use-using)