[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0
This commit is contained in:
EngineX CI
2026-07-30 09:35:51 +00:00
parent b4d01f481e
commit 56fd68e7dd
8871 changed files with 1454674 additions and 0 deletions

View File

@@ -0,0 +1,76 @@
#===----------------------------------------------------------------------===//
#
# Part of libcu++, the C++ Standard Library for your entire system,
# under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
#
#===----------------------------------------------------------------------===//
cmake_minimum_required(VERSION 3.18 FATAL_ERROR)
# Default to building for the GPU on the current system.
if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
set(CMAKE_CUDA_ARCHITECTURES native)
endif()
project(IMAGE_PIPELINE_EXAMPLE CUDA CXX)
# This example requires CUDA 13.1+ for the libcu++ runtime APIs.
find_package(CUDAToolkit REQUIRED)
if (CUDAToolkit_VERSION VERSION_LESS 13.1)
message(
STATUS
"Skipping image_pipeline example: requires CUDA 13.1+ (found ${CUDAToolkit_VERSION})"
)
return()
endif()
# This example uses the CMake Package Manager (CPM) to simplify fetching CCCL from GitHub
# For more information, see https://github.com/cpm-cmake/CPM.cmake
include(cmake/CPM.cmake)
# We define these as variables so they can be overridden in CI to pull from a PR instead of CCCL `main`
# In your project, these variables are unnecessary and you can just use the values directly
set(
CCCL_REPOSITORY
"https://github.com/NVIDIA/cccl"
CACHE STRING
"Git repository to fetch CCCL from"
)
set(CCCL_TAG "main" CACHE STRING "Git tag/branch to fetch from CCCL repository")
# This will automatically clone CCCL from GitHub and make the exported cmake targets available
CPMAddPackage(
NAME CCCL
GIT_REPOSITORY "${CCCL_REPOSITORY}"
GIT_TAG ${CCCL_TAG}
GIT_SHALLOW ON
)
# Image processing pipeline — a multi-file example showcasing the libcu++
# runtime APIs: device selection, memory pools, buffers, copy_bytes,
# fill_bytes, double-buffered streams, events, timed events, and CUB
# operations (histogram, transform, transform-reduce, block reduce).
add_executable(image_pipeline main.cu detail.cu)
target_include_directories(image_pipeline PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}")
target_compile_features(image_pipeline PRIVATE cuda_std_17)
target_link_libraries(image_pipeline PRIVATE CCCL::CCCL)
# Device lambdas (used in CUB transform/reduce ops) require extended lambda support.
target_compile_options(
image_pipeline
PRIVATE $<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:--extended-lambda>
)
option(
IMAGE_PIPELINE_ENABLE_RUNTIME_TEST
"Enable the image_pipeline CTest runtime test. Requires a GPU and several GB of host/device memory."
OFF
)
if (IMAGE_PIPELINE_ENABLE_RUNTIME_TEST)
include(CTest)
enable_testing()
add_test(NAME image_pipeline COMMAND image_pipeline)
endif()

View File

@@ -0,0 +1,53 @@
# Image Processing Pipeline Example
A multi-file example showcasing the CCCL Runtime and CUB APIs working together
in a semi-realistic tiled image processing pipeline.
## What it does
The example generates a synthetic 65K x 65K (~4 GB) grayscale space observation
on the GPU, then processes it in tiles that fit in GPU memory:
1. **Pass 1 - Histogram**: Upload each tile, compute per-tile histograms with
`cub::DeviceHistogram`, download and accumulate into a global histogram.
2. **Host interlude**: Compute Otsu's threshold (optimal foreground/background
split) and build a histogram equalization lookup table from the CDF.
3. **Pass 2 - Equalize + Analyze**: For each tile, apply the equalization LUT
(`cub::DeviceTransform`), normalize to float (`cub::DeviceTransform`),
compute thresholded count/min/max/sum (`cub::DeviceReduce::TransformReduce`),
and GPU-downscale a preview (`cub::BlockReduce`).
4. **Output**: Write `input_preview.bmp` and `equalized_preview.bmp` (1024 x
1024 previews). The equalized image reveals nebula structure and stars that
are barely visible in the dark original.
## CCCL APIs demonstrated
| File | APIs |
|------|------|
| `image_pipeline.h` | Shared constants and buffer/plan structs using `cuda::device_buffer`, `cuda::host_buffer`, `cuda::mr::shared_resource`, and spans |
| `detail.h` | Example-local declarations for image generation, preview output, reporting, and downscaling helpers |
| `detail.cu` | Synthetic image generation with `cuda::launch`/`cuda::distribute`, tile preview downscaling, `cuda::copy_bytes`, memory-pool statistics, and BMP output |
| `main.cu` | `cuda::devices`, `cuda::device_ref`, `cuda::device_attributes`, `cuda::arch_traits_for`, `cuda::device_memory_pool`, `cuda::memory_pool_properties`, `cuda::mr::shared_resource`, `cuda::make_buffer`, `cuda::make_pinned_buffer`, `cuda::copy_bytes`, `cuda::copy_configuration`, `cuda::fill_bytes`, `cuda::stream`, `cuda::timed_event`, `stream.wait(...)`, `buffer.first()`, `buffer.subspan()`, `buffer.get_unsynchronized()`, CUB `DeviceHistogram`, `DeviceTransform`, `DeviceReduce::TransformReduce`, and `BlockReduce` |
| `CMakeLists.txt` | Standalone CMake/CPM setup for consuming CCCL from a chosen repository and tag |
## Building and running
```bash
cd examples/image_pipeline
cmake -S . -B build -DCMAKE_CUDA_ARCHITECTURES=native
cmake --build build
./build/image_pipeline
```
The example requires CUDA Toolkit 13.1 or newer for the CCCL Runtime APIs used
by the sample. The standalone CMake project uses the vendored `cmake/CPM.cmake`
helper to fetch CCCL; override `CCCL_REPOSITORY` and `CCCL_TAG` to build against
a local checkout or a specific branch.
The example requires ~4 GB of pinned host memory for the full image and uses
60% of GPU memory for the per-tile working set. It should run on any GPU with
at least 4 GB of memory.

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,345 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
/// @file
/// Implementation of supporting details: synthetic image generation and
/// printing/output helpers. See detail.h for the interface.
#include <cuda/algorithm>
#include <cuda/cmath>
#include <cuda/launch>
#include <cuda/memory_pool>
#include <cuda/std/algorithm>
#include <cuda/std/span>
#include <cuda/stream>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <detail.h>
// ── Image generation ─────────────────────────────────────────────────
__device__ unsigned pixel_hash(int x, int y, unsigned seed)
{
unsigned h = static_cast<unsigned>(x) * 1103515245u + static_cast<unsigned>(y) * 12345u + seed;
h = (h ^ (h >> 16)) * 0x45d9f3bu;
h = (h ^ (h >> 13)) * 0x85ebca6bu;
return h ^ (h >> 16);
}
__device__ float hash01(int x, int y, unsigned seed)
{
return static_cast<float>(pixel_hash(x, y, seed) & 0xFFFF) / 65535.0f;
}
__device__ float value_noise(float fx, float fy, float freq, unsigned seed)
{
const float sx = fx * freq, sy = fy * freq;
const int ix = static_cast<int>(floorf(sx)), iy = static_cast<int>(floorf(sy));
float tx = sx - ix, ty = sy - iy;
tx = tx * tx * (3.0f - 2.0f * tx);
ty = ty * ty * (3.0f - 2.0f * ty);
const float a = hash01(ix, iy, seed) + (hash01(ix + 1, iy, seed) - hash01(ix, iy, seed)) * tx;
const float b = hash01(ix, iy + 1, seed) + (hash01(ix + 1, iy + 1, seed) - hash01(ix, iy + 1, seed)) * tx;
return a + (b - a) * ty;
}
__device__ float fbm(float fx, float fy, int octaves, float freq, unsigned seed)
{
float val = 0, amp = 1.0f, total = 0;
for (int i = 0; i < octaves; ++i)
{
val += amp * value_noise(fx, fy, freq, seed + static_cast<unsigned>(i) * 7919u);
total += amp;
amp *= 0.5f;
freq *= 2.0f;
}
return val / total;
}
struct generate_kernel
{
template <typename Config>
__device__ void operator()(Config config, cuda::std::span<pixel_t> out, int width, int row_offset)
{
const auto tid = cuda::gpu_thread.rank(cuda::grid, config);
if (tid >= out.size())
{
return;
}
const int gx = static_cast<int>(tid) % width;
const int gy = static_cast<int>(tid) / width + row_offset;
const float fx = static_cast<float>(gx) / image_width;
const float fy = static_cast<float>(gy) / image_height;
float val = 4.0f + 3.0f * fy;
const float noise = (hash01(gx, gy, 0u) - 0.5f) * 6.0f;
val += noise;
const float neb1_d = ((fx - 0.6f) * (fx - 0.6f) + (fy - 0.35f) * (fy - 0.35f)) / 0.06f;
float neb1 = 40.0f * expf(-neb1_d);
const float neb2_d = ((fx - 0.35f) * (fx - 0.35f) + (fy - 0.6f) * (fy - 0.6f)) / 0.03f;
float neb2 = 22.0f * expf(-neb2_d);
const float tex = fbm(fx, fy, 5, 8.0f, 42u);
neb1 *= (0.5f + tex);
neb2 *= (0.3f + 0.7f * tex);
const float dust = fbm(fx + 0.1f, fy, 4, 6.0f, 137u);
const float dust_mask = fmaxf(0.0f, 1.0f - 2.0f * fabsf(dust - 0.5f));
neb1 *= (1.0f - 0.6f * dust_mask * expf(-neb1_d * 2.0f));
val += neb1 + neb2;
constexpr int star_grid = 64;
const int cx = (gx / star_grid) * star_grid + star_grid / 2;
const int cy = (gy / star_grid) * star_grid + star_grid / 2;
for (int dy = -1; dy <= 1; ++dy)
{
for (int dx = -1; dx <= 1; ++dx)
{
const int scx = cx + dx * star_grid;
const int scy = cy + dy * star_grid;
const unsigned sh = pixel_hash(scx, scy, 9999u);
if (static_cast<float>(sh & 0xFFFF) / 65535.0f < 0.08f)
{
const float jx = static_cast<float>((sh >> 4) & 0xFF) / 255.0f - 0.5f;
const float jy = static_cast<float>((sh >> 12) & 0xFF) / 255.0f - 0.5f;
const float sx = scx + jx * star_grid;
const float sy = scy + jy * star_grid;
const float d2 = (gx - sx) * (gx - sx) + (gy - sy) * (gy - sy);
const float radius = 2.0f + static_cast<float>((sh >> 20) & 0xF);
const float bright = 80.0f + static_cast<float>((sh >> 24) & 0x7F);
val += bright * expf(-d2 / (2.0f * radius * radius));
}
}
}
out[tid] = static_cast<pixel_t>(fminf(255.0f, fmaxf(0.0f, val)));
}
};
// ── Image generation ─────────────────────────────────────────────────
void generate_image(cuda::stream_ref stream, tile_buffers& bufs, int num_tiles, cuda::std::span<pixel_t> host_preview)
{
std::cout << "=== Image generation (GPU) ===\n";
cuda::timed_event gen_start{stream};
for (int t = 0; t < num_tiles; ++t)
{
const size_t offset = static_cast<size_t>(t) * bufs.tile_pixels;
const size_t count = cuda::std::min(bufs.tile_pixels, image_pixels - offset);
const int tile_rows = static_cast<int>(count / image_width);
const int row_offset = t * static_cast<int>(bufs.tile_pixels / image_width);
constexpr int block_size = 256;
const auto config = cuda::distribute<block_size>(static_cast<int>(count));
cuda::launch(stream, config, generate_kernel{}, bufs.dev_tile[0].first(count), image_width, row_offset);
// Downscale the generated tile for the input preview while it's still on device.
downscale_tile(stream, bufs, 0, bufs.dev_tile[0].first(count), row_offset, tile_rows, host_preview);
// Copy generated tile to host for the histogram pass.
cuda::copy_bytes(stream, bufs.dev_tile[0].first(count), bufs.host_image.subspan(offset, count));
}
cuda::timed_event gen_end{stream};
stream.sync();
const double gen_ms = (gen_end - gen_start).count() / 1e6;
std::cout
<< " Generated " << image_width << 'x' << image_height << " space observation (" << std::fixed
<< std::setprecision(0) << image_pixels * sizeof(pixel_t) / (1024.0 * 1024.0) << " MB) in ~" << std::setprecision(1)
<< gen_ms << " ms\n\n"
<< std::defaultfloat << std::setprecision(6);
}
// ── Printing / output helpers ────────────────────────────────────────
void print_device_info(cuda::device_ref dev, cuda::arch_traits_t traits, size_t total_mem)
{
const auto name = dev.name();
const auto cc = dev.attribute(cuda::device_attributes::compute_capability);
std::cout << "\nSelected device " << dev.get() << ": ";
std::cout.write(name.data(), static_cast<std::streamsize>(name.size()));
std::cout
<< "\n Compute capability: " << cc.major_cap() << '.' << cc.minor_cap() << "\n Total memory : " << std::fixed
<< std::setprecision(0) << total_mem / (1024.0 * 1024.0) << " MB"
<< "\n Max threads/block : " << traits.max_threads_per_block
<< "\n Max shared memory : " << traits.max_shared_memory_per_block << " bytes\n"
<< std::defaultfloat << std::setprecision(6);
}
void print_tile_plan(int tile_rows, int tile_alignment, int num_tiles, size_t budget, size_t total_mem)
{
std::cout
<< "\nTile plan:\n"
<< " Image : " << image_width << " x " << image_height << " (" << std::fixed << std::setprecision(0)
<< image_pixels * sizeof(pixel_t) / (1024.0 * 1024.0) << " MB)\n"
<< " GPU budget : " << budget / (1024.0 * 1024.0) << " MB (60% of " << total_mem / (1024.0 * 1024.0)
<< " MB)\n"
<< " Tile rows : " << tile_rows << " (aligned to " << tile_alignment << ")\n"
<< " Number of tiles: " << num_tiles << "\n\n"
<< std::defaultfloat << std::setprecision(6);
}
void print_allocation_info(size_t device_total, size_t gpu_budget, size_t tile_pixels, int tile_rows)
{
std::cout
<< std::fixed << std::setprecision(1) << " Device pool: " << device_total / (1024.0 * 1024.0) << " MB (initial), "
<< gpu_budget / (1024.0 * 1024.0) << " MB (max)\n"
<< " Tile size: " << tile_pixels << " pixels (" << tile_rows << " rows x " << image_width << " cols)\n"
<< " Pinned host: image=" << image_pixels * sizeof(pixel_t) / (1024.0 * 1024.0)
<< " MB, histogram=" << num_bins * sizeof(int) << " bytes\n\n"
<< std::defaultfloat << std::setprecision(6);
}
void print_pool_stats(tile_buffers& bufs)
{
const auto reserved = bufs.device_pool.get().attribute(cuda::memory_pool_attributes::reserved_mem_current);
const auto used = bufs.device_pool.get().attribute(cuda::memory_pool_attributes::used_mem_current);
std::cout
<< std::fixed << std::setprecision(1) << " Device pool: reserved=" << reserved / (1024.0 * 1024.0)
<< " MB, used=" << used / (1024.0 * 1024.0) << " MB\n"
<< std::defaultfloat << std::setprecision(6);
}
iqr_result compute_iqr(cuda::std::span<const histogram_count_t> hist, size_t total)
{
auto find_percentile = [&](float pct) {
const auto target = static_cast<histogram_count_t>(static_cast<double>(total) * pct);
histogram_count_t cumulative{};
for (size_t i = 0; i < hist.size(); ++i)
{
cumulative += hist[i];
if (cumulative >= target)
{
return static_cast<int>(i);
}
}
return static_cast<int>(hist.size()) - 1;
};
return {find_percentile(0.25f), find_percentile(0.75f)};
}
void print_pass_stats(double ms, long long total_selected, double mean_selected, float global_min, float global_max)
{
// Note: times measured via cuda::timed_event are approximate GPU-side measurements.
std::cout
<< std::fixed << std::setprecision(1) << " Pass time: ~" << ms << " ms\n"
<< " Pixels above threshold: " << total_selected << " / " << image_pixels << " ("
<< 100.0 * total_selected / image_pixels << "%)\n"
<< std::setprecision(4) << " Selected range: [" << global_min << ", " << global_max << "], mean=" << mean_selected
<< '\n'
<< std::defaultfloat << std::setprecision(6);
}
void print_sanity_check(iqr_result orig, iqr_result eq)
{
const bool ok = eq.width() > orig.width();
std::cout
<< "=== Sanity check ===\n"
<< " Original IQR (25th-75th): [" << orig.p25 << ", " << orig.p75 << "] (span " << orig.width() << ")\n"
<< " Equalized IQR (25th-75th): [" << eq.p25 << ", " << eq.p75 << "] (span " << eq.width() << ")\n"
<< " Equalization spread distribution: " << (ok ? "YES" : "NO") << "\n\n";
}
void print_summary(int num_tiles, int tile_rows, double pass1_ms, double pass2_ms, bool ok)
{
std::cout
<< "=== Summary ===\n"
<< " Image: " << image_width << " x " << image_height << '\n'
<< " Tiles: " << num_tiles << " (" << tile_rows << " rows each)\n"
<< std::fixed << std::setprecision(1) << " Pass 1 (hist): ~" << pass1_ms << " ms\n"
<< " Pass 2 (stats): ~" << pass2_ms << " ms\n"
<< " Total pipeline: ~" << pass1_ms + pass2_ms << " ms\n"
<< " Result: " << (ok ? "PASSED" : "FAILED") << '\n'
<< std::defaultfloat << std::setprecision(6);
}
bool write_bmp(const char* filename, cuda::std::span<const pixel_t> data, int width, int height)
{
std::ofstream file(filename, std::ios::binary);
if (!file)
{
std::cerr << "Failed to open " << filename << " for writing\n";
return false;
}
// BMP rows must be padded to a 4-byte boundary.
const int row_stride = (width + 3) & ~3;
const int pixel_size = row_stride * height;
const int file_size = 54 + 256 * 4 + pixel_size; // header + palette + pixels
auto put2 = [&](int v) {
const char b[2] = {static_cast<char>(v & 0xFF), static_cast<char>((v >> 8) & 0xFF)};
file.write(b, 2);
};
auto put4 = [&](int v) {
const char b[4] = {static_cast<char>(v & 0xFF),
static_cast<char>((v >> 8) & 0xFF),
static_cast<char>((v >> 16) & 0xFF),
static_cast<char>((v >> 24) & 0xFF)};
file.write(b, 4);
};
// File header (14 bytes).
file.put('B');
file.put('M');
put4(file_size);
put4(0); // reserved
put4(54 + 256 * 4); // pixel data offset (after header + palette)
// DIB header (BITMAPINFOHEADER, 40 bytes).
put4(40); // header size
put4(width);
put4(height);
put2(1); // color planes
put2(8); // bits per pixel (8-bit indexed)
put4(0); // no compression
put4(pixel_size);
put4(2835); // horizontal resolution (72 DPI)
put4(2835); // vertical resolution
put4(256); // palette entries
put4(0); // all colors important
// Grayscale palette: 256 entries of (B, G, R, 0).
for (int i = 0; i < 256; ++i)
{
const char c = static_cast<char>(i);
file.put(c);
file.put(c);
file.put(c);
file.put(0);
}
// Pixel data — BMP stores rows bottom-to-top.
const char pad[3] = {0, 0, 0};
const int pad_bytes = row_stride - width;
for (int y = height - 1; y >= 0; --y)
{
file.write(reinterpret_cast<const char*>(&data[static_cast<size_t>(y) * width]), width);
if (pad_bytes > 0)
{
file.write(pad, pad_bytes);
}
}
if (!file)
{
std::cerr << "Failed to write " << filename << '\n';
return false;
}
std::cout << " Wrote " << filename << " (" << width << " x " << height << ")\n";
return true;
}

View File

@@ -0,0 +1,63 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef DETAIL_H
#define DETAIL_H
/// @file
/// Supporting details for the image pipeline example: synthetic image
/// generation and printing/output helpers. These are not part of the
/// pipeline itself — in a real application, image generation would be
/// replaced by actual data loading, and the printing would be replaced
/// by your application's logging.
#include <image_pipeline.h>
// ── Image generation ─────────────────────────────────────────────────
/// Generate a synthetic space observation on the GPU, tile by tile,
/// and produce a downscaled input preview.
/// In a real application this would be replaced by loading actual data.
void generate_image(cuda::stream_ref stream, tile_buffers& bufs, int num_tiles, cuda::std::span<pixel_t> host_preview);
// ── Printing / output helpers ────────────────────────────────────────
void print_device_info(cuda::device_ref dev, cuda::arch_traits_t traits, size_t total_mem);
void print_tile_plan(int tile_rows, int tile_alignment, int num_tiles, size_t budget, size_t total_mem);
void print_allocation_info(size_t device_total, size_t gpu_budget, size_t tile_pixels, int tile_rows);
void print_pool_stats(tile_buffers& bufs);
struct iqr_result
{
int p25, p75;
[[nodiscard]] int width() const noexcept
{
return p75 - p25;
}
};
[[nodiscard]] iqr_result compute_iqr(cuda::std::span<const histogram_count_t> hist, size_t total);
void print_pass_stats(double ms, long long total_selected, double mean_selected, float global_min, float global_max);
void print_sanity_check(iqr_result orig, iqr_result eq);
void print_summary(int num_tiles, int tile_rows, double pass1_ms, double pass2_ms, bool ok);
[[nodiscard]] bool write_bmp(const char* filename, cuda::std::span<const pixel_t> data, int width, int height);
/// Downscale a device tile using CUB BlockReduce and copy the result
/// to the host preview buffer.
void downscale_tile(
cuda::stream_ref stream,
tile_buffers& bufs,
int slot,
cuda::std::span<const pixel_t> dev_src,
int row_offset,
int tile_rows,
cuda::std::span<pixel_t> host_preview);
#endif // DETAIL_H

View File

@@ -0,0 +1,85 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef IMAGE_PIPELINE_H
#define IMAGE_PIPELINE_H
#include <cuda/buffer>
#include <cuda/devices>
#include <cuda/memory_resource>
#include <cuda/std/cstddef>
#include <cuda/std/cstdint>
#include <cuda/std/limits>
#include <cuda/std/span>
#include <cuda/stream>
// ── Constants ────────────────────────────────────────────────────────
/// Pixel type: 8-bit grayscale.
using pixel_t = uint8_t;
/// Full-image histogram count type.
using histogram_count_t = long long;
/// Image dimensions. 65K x 65K (~4 GB raw). Large enough that
/// the working set won't fit in most GPUs, forcing real tiling.
/// Note: the full image is held in pinned host memory (~4 GB).
/// Ensure the system has at least 8 GB of RAM.
inline constexpr int image_width = 65536;
inline constexpr int image_height = 65536;
inline constexpr size_t image_pixels = static_cast<size_t>(image_width) * image_height;
/// Preview downscale factor. The output preview images are
/// (image_width / preview_scale) x (image_height / preview_scale).
inline constexpr int preview_scale = 64;
/// Histogram bins (one per possible grayscale value).
inline constexpr int num_bins = cuda::std::numeric_limits<pixel_t>::max() + 1;
inline constexpr int num_levels = num_bins + 1; // CUB needs num_levels = num_bins + 1
// ── Shared data structures ───────────────────────────────────────────
/// Device selection and tile sizing results.
struct device_plan
{
cuda::device_ref device;
int tile_rows;
int num_tiles;
size_t gpu_budget;
};
/// All working memory for the pipeline.
struct tile_buffers
{
cuda::device_buffer<pixel_t> dev_tile[2]; // double-buffered H2D
cuda::device_buffer<float> dev_float_tile[2]; // normalized float tile
cuda::device_buffer<int> dev_histogram[2]; // per-tile histogram
cuda::device_buffer<float4> dev_tile_stats; // per-tile reduction: {count, min, max, sum}
cuda::device_buffer<pixel_t> dev_lut; // equalization LUT
cuda::device_buffer<pixel_t> dev_equalized[2]; // equalized tile
cuda::host_buffer<pixel_t> host_image; // full image in pinned memory
cuda::host_buffer<int> host_tile_histograms; // per-tile histograms
cuda::host_buffer<float4> host_tile_stats; // per-tile stats readback
cuda::device_buffer<pixel_t> dev_preview[2]; // downscaled preview tile
cuda::mr::shared_resource<cuda::device_memory_pool> device_pool;
size_t tile_pixels;
};
/// Per-tile processing statistics.
struct tile_stats
{
float min_val;
float max_val;
float sum;
long long num_selected;
};
#endif // IMAGE_PIPELINE_H

View File

@@ -0,0 +1,616 @@
//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
/**
* Image processing pipeline — all runtime API usage in one file.
*
* This file contains the complete pipeline:
* - Device selection and tile sizing
* - Memory pool creation and buffer allocation
* - Tile upload/download with copy_bytes and fill_bytes
* - CUB-based processing (histogram, equalization, thresholding, reduction)
* - GPU downscale with CUB BlockReduce
* - Double-buffered two-stream orchestration
*
* Supporting details (image generation, printing) are in detail.cu.
*/
#include <cub/block/block_reduce.cuh>
#include <cub/device/device_histogram.cuh>
#include <cub/device/device_reduce.cuh>
#include <cub/device/device_transform.cuh>
#include <cuda/algorithm>
#include <cuda/buffer>
#include <cuda/cmath>
#include <cuda/devices>
#include <cuda/launch>
#include <cuda/memory_pool>
#include <cuda/memory_resource>
#include <cuda/std/__exception/cuda_error.h>
#include <cuda/std/algorithm>
#include <cuda/std/array>
#include <cuda/std/execution>
#include <cuda/std/functional>
#include <cuda/std/limits>
#include <cuda/std/span>
#include <cuda/stream>
#include <exception>
#include <iomanip>
#include <iostream>
#include <cuda_runtime_api.h>
#include <detail.h>
#include <image_pipeline.h>
// ═════════════════════════════════════════════════════════════════════
// Device selection
// ═════════════════════════════════════════════════════════════════════
static device_plan select_device_and_plan()
{
std::cout << "=== Device selection ===\n";
cuda::device_ref best = cuda::devices[0];
size_t best_mem = 0;
for (auto dev : cuda::devices)
{
const size_t total_bytes = dev.attribute(cuda::device_attributes::total_global_memory);
const int sms = dev.attribute(cuda::device_attributes::multiprocessor_count);
const auto name = dev.name();
std::cout << " [" << dev.get() << "] ";
std::cout.write(name.data(), static_cast<std::streamsize>(name.size()));
std::cout
<< " " << std::setw(3) << sms << " SMs " << std::fixed << std::setprecision(0)
<< total_bytes / (1024.0 * 1024.0) << " MB\n"
<< std::defaultfloat << std::setprecision(6);
if (total_bytes > best_mem)
{
best = dev;
best_mem = total_bytes;
}
}
const auto cc = best.attribute(cuda::device_attributes::compute_capability);
const auto traits = cuda::arch_traits_for(cc);
print_device_info(best, traits, best_mem);
// Budget 60% of total GPU memory for the per-tile working set.
const size_t budget = static_cast<size_t>(best_mem * 0.60);
const size_t overhead = 128 * 1024 * 1024;
const size_t bytes_per_pixel = 4 * sizeof(pixel_t) + 2 * sizeof(float);
const size_t usable_budget = (budget > overhead) ? (budget - overhead) : 0;
const size_t budget_rows = usable_budget / bytes_per_pixel / image_width;
constexpr int tile_alignment = preview_scale;
const size_t max_launch_rows =
(static_cast<size_t>(cuda::std::numeric_limits<int>::max()) / image_width / tile_alignment) * tile_alignment;
const size_t max_tile_rows = cuda::std::min(static_cast<size_t>(image_height), max_launch_rows);
const size_t aligned_tile_rows = (budget_rows / tile_alignment) * tile_alignment;
const auto clamped_tile_rows =
cuda::std::clamp(aligned_tile_rows, static_cast<size_t>(tile_alignment), max_tile_rows);
const int tile_rows = static_cast<int>(clamped_tile_rows);
const int num_tiles = cuda::ceil_div(image_height, tile_rows);
print_tile_plan(tile_rows, tile_alignment, num_tiles, budget, best_mem);
return {best, tile_rows, num_tiles, budget};
}
// ═════════════════════════════════════════════════════════════════════
// Buffer allocation
// ═════════════════════════════════════════════════════════════════════
static tile_buffers
allocate_tile_buffers(cuda::stream_ref stream, cuda::device_ref device, int tile_rows, size_t gpu_budget, int num_tiles)
{
std::cout << "=== Tile buffer allocation ===\n";
const size_t tile_pixels = static_cast<size_t>(tile_rows) * image_width;
const size_t preview_px = (tile_rows / preview_scale) * (image_width / preview_scale);
// Single pool for our buffers and CUB temporaries.
const size_t device_total =
2 * tile_pixels * sizeof(pixel_t) // double-buffered pixel tiles
+ 2 * tile_pixels * sizeof(pixel_t) // double-buffered equalized tiles
+ 2 * tile_pixels * sizeof(float) // double-buffered normalized float tiles
+ sizeof(float4) // reduction output
+ num_bins * sizeof(pixel_t) // equalization LUT
+ 2 * num_bins * sizeof(int) // double-buffered histograms
+ 2 * preview_px * sizeof(pixel_t); // double-buffered preview tiles
cuda::memory_pool_properties props{};
props.initial_pool_size = device_total;
props.max_pool_size = gpu_budget;
auto device_pool = cuda::mr::shared_resource<cuda::device_memory_pool>(
cuda::std::in_place_type<cuda::device_memory_pool>, device, props);
// Device buffers — no_init since kernels/copies write before reading.
auto dev_tile_0 = cuda::make_buffer<pixel_t>(stream, device_pool, tile_pixels, cuda::no_init);
auto dev_tile_1 = cuda::make_buffer<pixel_t>(stream, device_pool, tile_pixels, cuda::no_init);
auto dev_float_0 = cuda::make_buffer<float>(stream, device_pool, tile_pixels, cuda::no_init);
auto dev_float_1 = cuda::make_buffer<float>(stream, device_pool, tile_pixels, cuda::no_init);
auto dev_hist_0 = cuda::make_buffer<int>(stream, device_pool, num_bins, cuda::no_init);
auto dev_hist_1 = cuda::make_buffer<int>(stream, device_pool, num_bins, cuda::no_init);
auto dev_tile_stats = cuda::make_buffer<float4>(stream, device_pool, num_tiles, cuda::no_init);
auto dev_lut = cuda::make_buffer<pixel_t>(stream, device_pool, num_bins, cuda::no_init);
auto dev_equalized_0 = cuda::make_buffer<pixel_t>(stream, device_pool, tile_pixels, cuda::no_init);
auto dev_equalized_1 = cuda::make_buffer<pixel_t>(stream, device_pool, tile_pixels, cuda::no_init);
auto dev_preview_0 = cuda::make_buffer<pixel_t>(stream, device_pool, preview_px, cuda::no_init);
auto dev_preview_1 = cuda::make_buffer<pixel_t>(stream, device_pool, preview_px, cuda::no_init);
// Pinned host buffers — make_pinned_buffer uses the default pinned pool.
auto host_image = cuda::make_pinned_buffer<pixel_t>(stream, image_pixels, pixel_t{0});
auto host_tile_hists = cuda::make_pinned_buffer<int>(stream, static_cast<size_t>(num_tiles) * num_bins, int{0});
auto host_tile_stats = cuda::make_pinned_buffer<float4>(stream, num_tiles, float4{0, 0, 0, 0});
print_allocation_info(device_total, gpu_budget, tile_pixels, tile_rows);
return {
{cuda::std::move(dev_tile_0), cuda::std::move(dev_tile_1)},
{cuda::std::move(dev_float_0), cuda::std::move(dev_float_1)},
{cuda::std::move(dev_hist_0), cuda::std::move(dev_hist_1)},
cuda::std::move(dev_tile_stats),
cuda::std::move(dev_lut),
{cuda::std::move(dev_equalized_0), cuda::std::move(dev_equalized_1)},
cuda::std::move(host_image),
cuda::std::move(host_tile_hists),
cuda::std::move(host_tile_stats),
{cuda::std::move(dev_preview_0), cuda::std::move(dev_preview_1)},
device_pool,
tile_pixels,
};
}
// ═════════════════════════════════════════════════════════════════════
// Tile transfer helpers
// ═════════════════════════════════════════════════════════════════════
static size_t upload_tile(cuda::stream_ref stream, tile_buffers& bufs, int slot, int tile_idx, int tile_rows)
{
const size_t offset = static_cast<size_t>(tile_idx) * bufs.tile_pixels;
const size_t count = cuda::std::min(bufs.tile_pixels, image_pixels - offset);
cuda::copy_configuration config{};
config.src_access_order = cuda::source_access_order::stream;
cuda::copy_bytes(stream, bufs.host_image.subspan(offset, count), bufs.dev_tile[slot].first(count), config);
cuda::fill_bytes(stream, bufs.dev_histogram[slot], uint8_t{0});
return count;
}
static void download_tile_histogram(cuda::stream_ref stream, tile_buffers& bufs, int slot, int tile_idx)
{
const size_t offset = static_cast<size_t>(tile_idx) * num_bins;
cuda::copy_bytes(stream, bufs.dev_histogram[slot], bufs.host_tile_histograms.subspan(offset, num_bins));
}
static void accumulate_histograms(tile_buffers& bufs, int num_tiles, cuda::std::span<histogram_count_t> result)
{
for (int i = 0; i < num_bins; ++i)
{
result[i] = 0;
}
for (int t = 0; t < num_tiles; ++t)
{
const size_t offset = static_cast<size_t>(t) * num_bins;
for (int i = 0; i < num_bins; ++i)
{
result[i] += bufs.host_tile_histograms.get_unsynchronized(offset + i);
}
}
}
static void upload_lut(cuda::stream_ref stream, tile_buffers& bufs, cuda::std::span<const pixel_t> host_lut)
{
cuda::copy_bytes(stream, host_lut, bufs.dev_lut);
}
// ═════════════════════════════════════════════════════════════════════
// CUB-based processing
// ═════════════════════════════════════════════════════════════════════
static auto make_cub_env(cuda::stream_ref stream, cuda::mr::shared_resource<cuda::device_memory_pool>& pool)
{
const auto mr_prop = cuda::std::execution::prop{cuda::mr::get_memory_resource_t{}, pool};
return cuda::std::execution::env{stream, mr_prop};
}
static void check_cub(cudaError_t err, const char* msg)
{
if (err != cudaSuccess)
{
throw cuda::cuda_error(err, msg, "CUB");
}
}
// ── Processing functions ─────────────────────────────────────────────
static void compute_histogram(cuda::stream_ref stream, tile_buffers& bufs, int slot, size_t tile_pixel_count)
{
auto env = make_cub_env(stream, bufs.device_pool);
check_cub(
cub::DeviceHistogram::HistogramEven(
bufs.dev_tile[slot].first(tile_pixel_count).data(),
bufs.dev_histogram[slot].data(),
num_levels,
0,
num_bins,
static_cast<int>(tile_pixel_count),
env),
"HistogramEven (pass 1)");
}
static void process_tile(
cuda::stream_ref stream, tile_buffers& bufs, int slot, int tile_idx, size_t tile_pixel_count, float threshold)
{
const int n = static_cast<int>(tile_pixel_count);
auto pixel_data = bufs.dev_tile[slot].first(tile_pixel_count).data();
auto eq_data = bufs.dev_equalized[slot].data();
auto float_data = bufs.dev_float_tile[slot].data();
auto env = make_cub_env(stream, bufs.device_pool);
// Equalize: apply the LUT to remap pixel intensities.
auto lut_span = bufs.dev_lut.first(num_bins);
auto equalize = [lut_span] __device__(pixel_t p) -> pixel_t {
return lut_span[p];
};
check_cub(cub::DeviceTransform::Transform(pixel_data, eq_data, n, equalize, env), "Transform (equalize)");
// Normalize: convert uint8 pixels to [0, 1] floats.
auto normalize = [] __device__(pixel_t p) -> float {
constexpr float pixel_max = cuda::std::numeric_limits<pixel_t>::max();
return static_cast<float>(p) / pixel_max;
};
check_cub(cub::DeviceTransform::Transform(eq_data, float_data, n, normalize, env), "Transform (normalize)");
check_cub(
cub::DeviceHistogram::HistogramEven(eq_data, bufs.dev_histogram[slot].data(), num_levels, 0, num_bins, n, env),
"HistogramEven (pass 2)");
// Combined threshold + count/min/max/sum in a single pass via float4: x=count, y=min, z=max, w=sum.
// Each tile writes to its own output index — no sync needed between tiles.
constexpr float flt_max = cuda::std::numeric_limits<float>::max();
constexpr float flt_low = cuda::std::numeric_limits<float>::lowest();
const float4 identity{0.0f, flt_max, flt_low, 0.0f};
auto threshold_stats = [threshold] __device__(float v) -> float4 {
if (v > threshold)
{
return {1.0f, v, v, v}; // count=1, min=v, max=v, sum=v
}
return {0.0f, flt_max, flt_low, 0.0f};
};
auto stats_reduce = [] __device__(float4 a, float4 b) -> float4 {
return {a.x + b.x, cuda::std::min(a.y, b.y), cuda::std::max(a.z, b.z), a.w + b.w};
};
check_cub(cub::DeviceReduce::TransformReduce(
float_data, bufs.dev_tile_stats.data() + tile_idx, n, stats_reduce, threshold_stats, identity, env),
"TransformReduce");
// D2H copy into this tile's slot — no sync, read after all tiles finish.
cuda::copy_bytes(stream, bufs.dev_tile_stats.subspan(tile_idx, 1), bufs.host_tile_stats.subspan(tile_idx, 1));
}
/// Accumulate per-tile stats into a single result after all tiles finish.
static tile_stats accumulate_tile_stats(tile_buffers& bufs, int num_tiles)
{
tile_stats result{};
result.min_val = cuda::std::numeric_limits<float>::max();
result.max_val = cuda::std::numeric_limits<float>::lowest();
for (int t = 0; t < num_tiles; ++t)
{
const auto s = bufs.host_tile_stats.get_unsynchronized(t);
result.num_selected += static_cast<long long>(s.x);
result.min_val = cuda::std::min(result.min_val, s.y);
result.max_val = cuda::std::max(result.max_val, s.z);
result.sum += s.w;
}
return result;
}
// ── Host-side algorithms ─────────────────────────────────────────────
static float compute_otsu_threshold(cuda::std::span<const histogram_count_t> histogram, size_t total_pixels)
{
double total_sum = 0;
for (int i = 0; i < num_bins; ++i)
{
total_sum += static_cast<double>(i) * histogram[i];
}
double sum_bg = 0, weight_bg = 0, max_var = 0;
int best_t = 0;
for (int t = 0; t < num_bins; ++t)
{
weight_bg += histogram[t];
if (weight_bg == 0)
{
continue;
}
const double weight_fg = static_cast<double>(total_pixels) - weight_bg;
if (weight_fg == 0)
{
break;
}
sum_bg += static_cast<double>(t) * histogram[t];
const double mean_bg = sum_bg / weight_bg;
const double mean_fg = (total_sum - sum_bg) / weight_fg;
const double var = weight_bg * weight_fg * (mean_bg - mean_fg) * (mean_bg - mean_fg);
if (var > max_var)
{
max_var = var;
best_t = t;
}
}
return static_cast<float>(best_t) / cuda::std::numeric_limits<pixel_t>::max();
}
static void build_equalization_lut(
cuda::std::span<const histogram_count_t> histogram, size_t total_pixels, cuda::std::span<pixel_t> lut_out)
{
constexpr double max_val = cuda::std::numeric_limits<pixel_t>::max();
const double scale = max_val / static_cast<double>(total_pixels);
double cdf = 0;
for (int i = 0; i < num_bins; ++i)
{
cdf += histogram[i];
lut_out[i] = static_cast<pixel_t>(cuda::std::min(cdf * scale, max_val));
}
}
// ═════════════════════════════════════════════════════════════════════
// Downscale kernel
// ═════════════════════════════════════════════════════════════════════
// Each block produces one output pixel by box-averaging a scale×scale
// source block. Threads cooperatively load source pixels and sum
// them locally, then cub::BlockReduce merges the partial sums.
//
// The block size is extracted from the launch config at compile time
// via cuda::gpu_thread.count(cuda::block, config), which is then used
// as the template parameter for cub::BlockReduce.
struct downscale_kernel
{
template <typename Config>
__device__ void
operator()(Config config, cuda::std::span<const pixel_t> src, cuda::std::span<pixel_t> dst, int src_width, int scale)
{
constexpr int block_size = cuda::gpu_thread.count(cuda::block, config);
const int out_idx = blockIdx.x;
if (out_idx >= static_cast<int>(dst.size()))
{
return;
}
const int dst_width = src_width / scale;
const int px = out_idx % dst_width;
const int py = out_idx / dst_width;
const int total_elems = scale * scale;
// Each thread sums its share of the scale×scale source block.
int local_sum = 0;
const int tid = threadIdx.x;
for (int i = tid; i < total_elems; i += block_size)
{
const int dy = i / scale;
const int dx = i % scale;
local_sum += src[static_cast<size_t>(py * scale + dy) * src_width + (px * scale + dx)];
}
// cub::BlockReduce sums the per-thread partial sums into a single
// block-wide total. Without CUB, this would be a manual shared-
// memory tree reduction:
//
// __shared__ int smem[block_size];
// smem[tid] = local_sum;
// __syncthreads();
// for (int s = block_size / 2; s > 0; s >>= 1)
// {
// if (tid < s) smem[tid] += smem[tid + s];
// __syncthreads();
// }
// int block_sum = smem[0];
using BlockReduceT = cub::BlockReduce<int, block_size>;
__shared__ typename BlockReduceT::TempStorage temp_storage;
const int block_sum = BlockReduceT(temp_storage).Sum(local_sum);
if (tid == 0)
{
dst[out_idx] = static_cast<pixel_t>(block_sum / total_elems);
}
}
};
void downscale_tile(
cuda::stream_ref stream,
tile_buffers& bufs,
int slot,
cuda::std::span<const pixel_t> dev_src,
int row_offset,
int tile_rows,
cuda::std::span<pixel_t> host_preview)
{
const int dst_rows = tile_rows / preview_scale;
const int dst_cols = image_width / preview_scale;
const int dst_pixels = dst_rows * dst_cols;
if (dst_pixels == 0)
{
return;
}
constexpr int block_size = 256;
const auto config = cuda::make_config(cuda::block_dims<block_size>(), cuda::grid_dims(dst_pixels));
cuda::launch(
stream,
config,
downscale_kernel{},
dev_src,
bufs.dev_preview[slot].first(static_cast<size_t>(dst_pixels)),
image_width,
preview_scale);
const int preview_row_offset = row_offset / preview_scale;
cuda::copy_bytes(
stream,
bufs.dev_preview[slot].first(static_cast<size_t>(dst_pixels)),
host_preview.subspan(static_cast<size_t>(preview_row_offset) * dst_cols, static_cast<size_t>(dst_pixels)));
}
// ═════════════════════════════════════════════════════════════════════
// Main
// ═════════════════════════════════════════════════════════════════════
int main()
try
{
// ── 1. Device selection and tile sizing ────────────────────────────
const auto plan = select_device_and_plan();
// ── 2. Allocate all buffers ────────────────────────────────────────
// Two streams for double-buffered tile processing. stream_a also
// handles setup work (allocation, LUT upload, etc.) between passes.
cuda::stream stream_a{plan.device};
cuda::stream stream_b{plan.device};
cuda::stream_ref streams[2] = {stream_a, stream_b};
auto bufs = allocate_tile_buffers(stream_a, plan.device, plan.tile_rows, plan.gpu_budget, plan.num_tiles);
// ── 3. Generate image and downscale input preview ──────────────────
const int pw = image_width / preview_scale;
const int ph = image_height / preview_scale;
auto host_input_preview = cuda::make_pinned_buffer<pixel_t>(stream_a, static_cast<size_t>(pw) * ph, pixel_t{0});
generate_image(stream_a, bufs, plan.num_tiles, host_input_preview.subspan(0));
bool outputs_ok = write_bmp("input_preview.bmp", host_input_preview.subspan(0), pw, ph);
// ── 4. Pass 1: histogram (double-buffered) ─────────────────────────
// stream_a: [upload tile 0] [histogram 0] [download 0] [tile 2] ...
// stream_b: [upload tile 1] [histogram 1] [download 1] ...
std::cout << "=== Pass 1: histogram ===\n";
cuda::timed_event pass1_start{stream_a};
for (int t = 0; t < plan.num_tiles; ++t)
{
const int slot = t % 2;
const size_t count = upload_tile(streams[slot], bufs, slot, t, plan.tile_rows);
compute_histogram(streams[slot], bufs, slot, count);
download_tile_histogram(streams[slot], bufs, slot, t);
}
stream_a.wait(stream_b);
cuda::timed_event pass1_end{stream_a};
stream_a.sync();
const double pass1_ms = (pass1_end - pass1_start).count() / 1e6;
std::cout << std::fixed << std::setprecision(1) << " Histogram pass: " << pass1_ms << " ms\n"
<< std::defaultfloat << std::setprecision(6);
// ── 5. Otsu threshold + equalization LUT ───────────────────────────
cuda::std::array<histogram_count_t, num_bins> original_hist{};
cuda::std::span global_hist_span{original_hist};
accumulate_histograms(bufs, plan.num_tiles, global_hist_span);
const float otsu = compute_otsu_threshold(global_hist_span, image_pixels);
std::cout
<< std::fixed << std::setprecision(4) << " Otsu threshold: " << otsu << " (" << static_cast<int>(otsu * 255)
<< " / 255)\n"
<< std::defaultfloat << std::setprecision(6);
pixel_t host_lut[num_bins];
build_equalization_lut(global_hist_span, image_pixels, cuda::std::span<pixel_t>(host_lut, num_bins));
// Construct a pinned buffer from the host LUT array — the buffer
// copies the data in stream order, no manual sync needed.
auto pinned_lut = cuda::make_pinned_buffer<pixel_t>(stream_a, host_lut, host_lut + num_bins);
upload_lut(stream_a, bufs, pinned_lut.subspan(0));
std::cout << " Equalization LUT uploaded\n\n";
cuda::fill_bytes(stream_a, bufs.host_tile_histograms, uint8_t{0});
auto host_eq_preview = cuda::make_pinned_buffer<pixel_t>(stream_a, static_cast<size_t>(pw) * ph, pixel_t{0});
// stream_b must wait for the setup work on stream_a before starting pass 2.
stream_b.wait(stream_a);
// ── 6. Pass 2: equalize + threshold + stats + preview ──────────────
std::cout << "=== Pass 2: equalize + threshold + statistics ===\n";
cuda::timed_event pass2_start{stream_a};
for (int t = 0; t < plan.num_tiles; ++t)
{
const int slot = t % 2;
const size_t count = upload_tile(streams[slot], bufs, slot, t, plan.tile_rows);
process_tile(streams[slot], bufs, slot, t, count, otsu);
const int tile_rows = static_cast<int>(count / image_width);
const int row_offset = t * static_cast<int>(bufs.tile_pixels / image_width);
downscale_tile(
streams[slot],
bufs,
slot,
bufs.dev_equalized[slot].first(count),
row_offset,
tile_rows,
host_eq_preview.subspan(0));
download_tile_histogram(streams[slot], bufs, slot, t);
}
// Single sync after all tiles — stats, histograms, and preview are on host.
stream_a.wait(stream_b);
cuda::timed_event pass2_end{stream_a};
stream_a.sync();
const double pass2_ms = (pass2_end - pass2_start).count() / 1e6;
const auto stats = accumulate_tile_stats(bufs, plan.num_tiles);
const double mean_selected = (stats.num_selected > 0) ? static_cast<double>(stats.sum) / stats.num_selected : 0.0;
print_pass_stats(pass2_ms, stats.num_selected, mean_selected, stats.min_val, stats.max_val);
print_pool_stats(bufs);
// ── 7. Write equalized preview ─────────────────────────────────────
outputs_ok = write_bmp("equalized_preview.bmp", host_eq_preview.subspan(0), pw, ph) && outputs_ok;
if (!outputs_ok)
{
std::cerr << "One or more preview BMP files were not written.\n";
}
std::cout << '\n';
// ── 8. Sanity check ────────────────────────────────────────────────
const auto orig_iqr = compute_iqr(cuda::std::span{original_hist}, image_pixels);
cuda::std::array<histogram_count_t, num_bins> equalized_hist{};
accumulate_histograms(bufs, plan.num_tiles, cuda::std::span{equalized_hist});
const auto eq_iqr = compute_iqr(cuda::std::span{equalized_hist}, image_pixels);
print_sanity_check(orig_iqr, eq_iqr);
const bool ok = outputs_ok && eq_iqr.width() > orig_iqr.width();
print_summary(plan.num_tiles, plan.tile_rows, pass1_ms, pass2_ms, ok);
return ok ? 0 : 1;
}
catch (const cuda::cuda_error& e)
{
std::cerr << "CUDA error: " << e.what() << '\n';
return 1;
}
catch (const std::exception& e)
{
std::cerr << "Error: " << e.what() << '\n';
return 1;
}
catch (...)
{
std::cerr << "An unknown error was encountered\n";
return 1;
}