[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
76
cccl_upstream/examples/image_pipeline/CMakeLists.txt
Normal file
76
cccl_upstream/examples/image_pipeline/CMakeLists.txt
Normal file
@@ -0,0 +1,76 @@
|
||||
#===----------------------------------------------------------------------===//
|
||||
#
|
||||
# Part of libcu++, the C++ Standard Library for your entire system,
|
||||
# under the Apache License v2.0 with LLVM Exceptions.
|
||||
# See https://llvm.org/LICENSE.txt for license information.
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
#
|
||||
#===----------------------------------------------------------------------===//
|
||||
|
||||
cmake_minimum_required(VERSION 3.18 FATAL_ERROR)
|
||||
|
||||
# Default to building for the GPU on the current system.
|
||||
if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
|
||||
set(CMAKE_CUDA_ARCHITECTURES native)
|
||||
endif()
|
||||
|
||||
project(IMAGE_PIPELINE_EXAMPLE CUDA CXX)
|
||||
|
||||
# This example requires CUDA 13.1+ for the libcu++ runtime APIs.
|
||||
find_package(CUDAToolkit REQUIRED)
|
||||
if (CUDAToolkit_VERSION VERSION_LESS 13.1)
|
||||
message(
|
||||
STATUS
|
||||
"Skipping image_pipeline example: requires CUDA 13.1+ (found ${CUDAToolkit_VERSION})"
|
||||
)
|
||||
return()
|
||||
endif()
|
||||
|
||||
# This example uses the CMake Package Manager (CPM) to simplify fetching CCCL from GitHub
|
||||
# For more information, see https://github.com/cpm-cmake/CPM.cmake
|
||||
include(cmake/CPM.cmake)
|
||||
|
||||
# We define these as variables so they can be overridden in CI to pull from a PR instead of CCCL `main`
|
||||
# In your project, these variables are unnecessary and you can just use the values directly
|
||||
set(
|
||||
CCCL_REPOSITORY
|
||||
"https://github.com/NVIDIA/cccl"
|
||||
CACHE STRING
|
||||
"Git repository to fetch CCCL from"
|
||||
)
|
||||
set(CCCL_TAG "main" CACHE STRING "Git tag/branch to fetch from CCCL repository")
|
||||
|
||||
# This will automatically clone CCCL from GitHub and make the exported cmake targets available
|
||||
CPMAddPackage(
|
||||
NAME CCCL
|
||||
GIT_REPOSITORY "${CCCL_REPOSITORY}"
|
||||
GIT_TAG ${CCCL_TAG}
|
||||
GIT_SHALLOW ON
|
||||
)
|
||||
|
||||
# Image processing pipeline — a multi-file example showcasing the libcu++
|
||||
# runtime APIs: device selection, memory pools, buffers, copy_bytes,
|
||||
# fill_bytes, double-buffered streams, events, timed events, and CUB
|
||||
# operations (histogram, transform, transform-reduce, block reduce).
|
||||
add_executable(image_pipeline main.cu detail.cu)
|
||||
target_include_directories(image_pipeline PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}")
|
||||
target_compile_features(image_pipeline PRIVATE cuda_std_17)
|
||||
target_link_libraries(image_pipeline PRIVATE CCCL::CCCL)
|
||||
# Device lambdas (used in CUB transform/reduce ops) require extended lambda support.
|
||||
target_compile_options(
|
||||
image_pipeline
|
||||
PRIVATE $<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:--extended-lambda>
|
||||
)
|
||||
|
||||
option(
|
||||
IMAGE_PIPELINE_ENABLE_RUNTIME_TEST
|
||||
"Enable the image_pipeline CTest runtime test. Requires a GPU and several GB of host/device memory."
|
||||
OFF
|
||||
)
|
||||
|
||||
if (IMAGE_PIPELINE_ENABLE_RUNTIME_TEST)
|
||||
include(CTest)
|
||||
enable_testing()
|
||||
add_test(NAME image_pipeline COMMAND image_pipeline)
|
||||
endif()
|
||||
53
cccl_upstream/examples/image_pipeline/README.md
Normal file
53
cccl_upstream/examples/image_pipeline/README.md
Normal file
@@ -0,0 +1,53 @@
|
||||
|
||||
# Image Processing Pipeline Example
|
||||
|
||||
A multi-file example showcasing the CCCL Runtime and CUB APIs working together
|
||||
in a semi-realistic tiled image processing pipeline.
|
||||
|
||||
## What it does
|
||||
|
||||
The example generates a synthetic 65K x 65K (~4 GB) grayscale space observation
|
||||
on the GPU, then processes it in tiles that fit in GPU memory:
|
||||
|
||||
1. **Pass 1 - Histogram**: Upload each tile, compute per-tile histograms with
|
||||
`cub::DeviceHistogram`, download and accumulate into a global histogram.
|
||||
|
||||
2. **Host interlude**: Compute Otsu's threshold (optimal foreground/background
|
||||
split) and build a histogram equalization lookup table from the CDF.
|
||||
|
||||
3. **Pass 2 - Equalize + Analyze**: For each tile, apply the equalization LUT
|
||||
(`cub::DeviceTransform`), normalize to float (`cub::DeviceTransform`),
|
||||
compute thresholded count/min/max/sum (`cub::DeviceReduce::TransformReduce`),
|
||||
and GPU-downscale a preview (`cub::BlockReduce`).
|
||||
|
||||
4. **Output**: Write `input_preview.bmp` and `equalized_preview.bmp` (1024 x
|
||||
1024 previews). The equalized image reveals nebula structure and stars that
|
||||
are barely visible in the dark original.
|
||||
|
||||
## CCCL APIs demonstrated
|
||||
|
||||
| File | APIs |
|
||||
|------|------|
|
||||
| `image_pipeline.h` | Shared constants and buffer/plan structs using `cuda::device_buffer`, `cuda::host_buffer`, `cuda::mr::shared_resource`, and spans |
|
||||
| `detail.h` | Example-local declarations for image generation, preview output, reporting, and downscaling helpers |
|
||||
| `detail.cu` | Synthetic image generation with `cuda::launch`/`cuda::distribute`, tile preview downscaling, `cuda::copy_bytes`, memory-pool statistics, and BMP output |
|
||||
| `main.cu` | `cuda::devices`, `cuda::device_ref`, `cuda::device_attributes`, `cuda::arch_traits_for`, `cuda::device_memory_pool`, `cuda::memory_pool_properties`, `cuda::mr::shared_resource`, `cuda::make_buffer`, `cuda::make_pinned_buffer`, `cuda::copy_bytes`, `cuda::copy_configuration`, `cuda::fill_bytes`, `cuda::stream`, `cuda::timed_event`, `stream.wait(...)`, `buffer.first()`, `buffer.subspan()`, `buffer.get_unsynchronized()`, CUB `DeviceHistogram`, `DeviceTransform`, `DeviceReduce::TransformReduce`, and `BlockReduce` |
|
||||
| `CMakeLists.txt` | Standalone CMake/CPM setup for consuming CCCL from a chosen repository and tag |
|
||||
|
||||
## Building and running
|
||||
|
||||
```bash
|
||||
cd examples/image_pipeline
|
||||
cmake -S . -B build -DCMAKE_CUDA_ARCHITECTURES=native
|
||||
cmake --build build
|
||||
./build/image_pipeline
|
||||
```
|
||||
|
||||
The example requires CUDA Toolkit 13.1 or newer for the CCCL Runtime APIs used
|
||||
by the sample. The standalone CMake project uses the vendored `cmake/CPM.cmake`
|
||||
helper to fetch CCCL; override `CCCL_REPOSITORY` and `CCCL_TAG` to build against
|
||||
a local checkout or a specific branch.
|
||||
|
||||
The example requires ~4 GB of pinned host memory for the full image and uses
|
||||
60% of GPU memory for the per-tile working set. It should run on any GPU with
|
||||
at least 4 GB of memory.
|
||||
1297
cccl_upstream/examples/image_pipeline/cmake/CPM.cmake
Normal file
1297
cccl_upstream/examples/image_pipeline/cmake/CPM.cmake
Normal file
File diff suppressed because it is too large
Load Diff
345
cccl_upstream/examples/image_pipeline/detail.cu
Normal file
345
cccl_upstream/examples/image_pipeline/detail.cu
Normal file
@@ -0,0 +1,345 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
/// @file
|
||||
/// Implementation of supporting details: synthetic image generation and
|
||||
/// printing/output helpers. See detail.h for the interface.
|
||||
|
||||
#include <cuda/algorithm>
|
||||
#include <cuda/cmath>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/memory_pool>
|
||||
#include <cuda/std/algorithm>
|
||||
#include <cuda/std/span>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <fstream>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
|
||||
#include <detail.h>
|
||||
|
||||
// ── Image generation ─────────────────────────────────────────────────
|
||||
|
||||
__device__ unsigned pixel_hash(int x, int y, unsigned seed)
|
||||
{
|
||||
unsigned h = static_cast<unsigned>(x) * 1103515245u + static_cast<unsigned>(y) * 12345u + seed;
|
||||
h = (h ^ (h >> 16)) * 0x45d9f3bu;
|
||||
h = (h ^ (h >> 13)) * 0x85ebca6bu;
|
||||
return h ^ (h >> 16);
|
||||
}
|
||||
|
||||
__device__ float hash01(int x, int y, unsigned seed)
|
||||
{
|
||||
return static_cast<float>(pixel_hash(x, y, seed) & 0xFFFF) / 65535.0f;
|
||||
}
|
||||
|
||||
__device__ float value_noise(float fx, float fy, float freq, unsigned seed)
|
||||
{
|
||||
const float sx = fx * freq, sy = fy * freq;
|
||||
const int ix = static_cast<int>(floorf(sx)), iy = static_cast<int>(floorf(sy));
|
||||
float tx = sx - ix, ty = sy - iy;
|
||||
tx = tx * tx * (3.0f - 2.0f * tx);
|
||||
ty = ty * ty * (3.0f - 2.0f * ty);
|
||||
const float a = hash01(ix, iy, seed) + (hash01(ix + 1, iy, seed) - hash01(ix, iy, seed)) * tx;
|
||||
const float b = hash01(ix, iy + 1, seed) + (hash01(ix + 1, iy + 1, seed) - hash01(ix, iy + 1, seed)) * tx;
|
||||
return a + (b - a) * ty;
|
||||
}
|
||||
|
||||
__device__ float fbm(float fx, float fy, int octaves, float freq, unsigned seed)
|
||||
{
|
||||
float val = 0, amp = 1.0f, total = 0;
|
||||
for (int i = 0; i < octaves; ++i)
|
||||
{
|
||||
val += amp * value_noise(fx, fy, freq, seed + static_cast<unsigned>(i) * 7919u);
|
||||
total += amp;
|
||||
amp *= 0.5f;
|
||||
freq *= 2.0f;
|
||||
}
|
||||
return val / total;
|
||||
}
|
||||
|
||||
struct generate_kernel
|
||||
{
|
||||
template <typename Config>
|
||||
__device__ void operator()(Config config, cuda::std::span<pixel_t> out, int width, int row_offset)
|
||||
{
|
||||
const auto tid = cuda::gpu_thread.rank(cuda::grid, config);
|
||||
if (tid >= out.size())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const int gx = static_cast<int>(tid) % width;
|
||||
const int gy = static_cast<int>(tid) / width + row_offset;
|
||||
const float fx = static_cast<float>(gx) / image_width;
|
||||
const float fy = static_cast<float>(gy) / image_height;
|
||||
|
||||
float val = 4.0f + 3.0f * fy;
|
||||
const float noise = (hash01(gx, gy, 0u) - 0.5f) * 6.0f;
|
||||
val += noise;
|
||||
|
||||
const float neb1_d = ((fx - 0.6f) * (fx - 0.6f) + (fy - 0.35f) * (fy - 0.35f)) / 0.06f;
|
||||
float neb1 = 40.0f * expf(-neb1_d);
|
||||
const float neb2_d = ((fx - 0.35f) * (fx - 0.35f) + (fy - 0.6f) * (fy - 0.6f)) / 0.03f;
|
||||
float neb2 = 22.0f * expf(-neb2_d);
|
||||
|
||||
const float tex = fbm(fx, fy, 5, 8.0f, 42u);
|
||||
neb1 *= (0.5f + tex);
|
||||
neb2 *= (0.3f + 0.7f * tex);
|
||||
|
||||
const float dust = fbm(fx + 0.1f, fy, 4, 6.0f, 137u);
|
||||
const float dust_mask = fmaxf(0.0f, 1.0f - 2.0f * fabsf(dust - 0.5f));
|
||||
neb1 *= (1.0f - 0.6f * dust_mask * expf(-neb1_d * 2.0f));
|
||||
val += neb1 + neb2;
|
||||
|
||||
constexpr int star_grid = 64;
|
||||
const int cx = (gx / star_grid) * star_grid + star_grid / 2;
|
||||
const int cy = (gy / star_grid) * star_grid + star_grid / 2;
|
||||
for (int dy = -1; dy <= 1; ++dy)
|
||||
{
|
||||
for (int dx = -1; dx <= 1; ++dx)
|
||||
{
|
||||
const int scx = cx + dx * star_grid;
|
||||
const int scy = cy + dy * star_grid;
|
||||
const unsigned sh = pixel_hash(scx, scy, 9999u);
|
||||
if (static_cast<float>(sh & 0xFFFF) / 65535.0f < 0.08f)
|
||||
{
|
||||
const float jx = static_cast<float>((sh >> 4) & 0xFF) / 255.0f - 0.5f;
|
||||
const float jy = static_cast<float>((sh >> 12) & 0xFF) / 255.0f - 0.5f;
|
||||
const float sx = scx + jx * star_grid;
|
||||
const float sy = scy + jy * star_grid;
|
||||
const float d2 = (gx - sx) * (gx - sx) + (gy - sy) * (gy - sy);
|
||||
const float radius = 2.0f + static_cast<float>((sh >> 20) & 0xF);
|
||||
const float bright = 80.0f + static_cast<float>((sh >> 24) & 0x7F);
|
||||
val += bright * expf(-d2 / (2.0f * radius * radius));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out[tid] = static_cast<pixel_t>(fminf(255.0f, fmaxf(0.0f, val)));
|
||||
}
|
||||
};
|
||||
|
||||
// ── Image generation ─────────────────────────────────────────────────
|
||||
|
||||
void generate_image(cuda::stream_ref stream, tile_buffers& bufs, int num_tiles, cuda::std::span<pixel_t> host_preview)
|
||||
{
|
||||
std::cout << "=== Image generation (GPU) ===\n";
|
||||
cuda::timed_event gen_start{stream};
|
||||
|
||||
for (int t = 0; t < num_tiles; ++t)
|
||||
{
|
||||
const size_t offset = static_cast<size_t>(t) * bufs.tile_pixels;
|
||||
const size_t count = cuda::std::min(bufs.tile_pixels, image_pixels - offset);
|
||||
const int tile_rows = static_cast<int>(count / image_width);
|
||||
const int row_offset = t * static_cast<int>(bufs.tile_pixels / image_width);
|
||||
|
||||
constexpr int block_size = 256;
|
||||
const auto config = cuda::distribute<block_size>(static_cast<int>(count));
|
||||
cuda::launch(stream, config, generate_kernel{}, bufs.dev_tile[0].first(count), image_width, row_offset);
|
||||
|
||||
// Downscale the generated tile for the input preview while it's still on device.
|
||||
downscale_tile(stream, bufs, 0, bufs.dev_tile[0].first(count), row_offset, tile_rows, host_preview);
|
||||
|
||||
// Copy generated tile to host for the histogram pass.
|
||||
cuda::copy_bytes(stream, bufs.dev_tile[0].first(count), bufs.host_image.subspan(offset, count));
|
||||
}
|
||||
|
||||
cuda::timed_event gen_end{stream};
|
||||
stream.sync();
|
||||
const double gen_ms = (gen_end - gen_start).count() / 1e6;
|
||||
std::cout
|
||||
<< " Generated " << image_width << 'x' << image_height << " space observation (" << std::fixed
|
||||
<< std::setprecision(0) << image_pixels * sizeof(pixel_t) / (1024.0 * 1024.0) << " MB) in ~" << std::setprecision(1)
|
||||
<< gen_ms << " ms\n\n"
|
||||
<< std::defaultfloat << std::setprecision(6);
|
||||
}
|
||||
|
||||
// ── Printing / output helpers ────────────────────────────────────────
|
||||
|
||||
void print_device_info(cuda::device_ref dev, cuda::arch_traits_t traits, size_t total_mem)
|
||||
{
|
||||
const auto name = dev.name();
|
||||
const auto cc = dev.attribute(cuda::device_attributes::compute_capability);
|
||||
std::cout << "\nSelected device " << dev.get() << ": ";
|
||||
std::cout.write(name.data(), static_cast<std::streamsize>(name.size()));
|
||||
std::cout
|
||||
<< "\n Compute capability: " << cc.major_cap() << '.' << cc.minor_cap() << "\n Total memory : " << std::fixed
|
||||
<< std::setprecision(0) << total_mem / (1024.0 * 1024.0) << " MB"
|
||||
<< "\n Max threads/block : " << traits.max_threads_per_block
|
||||
<< "\n Max shared memory : " << traits.max_shared_memory_per_block << " bytes\n"
|
||||
<< std::defaultfloat << std::setprecision(6);
|
||||
}
|
||||
|
||||
void print_tile_plan(int tile_rows, int tile_alignment, int num_tiles, size_t budget, size_t total_mem)
|
||||
{
|
||||
std::cout
|
||||
<< "\nTile plan:\n"
|
||||
<< " Image : " << image_width << " x " << image_height << " (" << std::fixed << std::setprecision(0)
|
||||
<< image_pixels * sizeof(pixel_t) / (1024.0 * 1024.0) << " MB)\n"
|
||||
<< " GPU budget : " << budget / (1024.0 * 1024.0) << " MB (60% of " << total_mem / (1024.0 * 1024.0)
|
||||
<< " MB)\n"
|
||||
<< " Tile rows : " << tile_rows << " (aligned to " << tile_alignment << ")\n"
|
||||
<< " Number of tiles: " << num_tiles << "\n\n"
|
||||
<< std::defaultfloat << std::setprecision(6);
|
||||
}
|
||||
|
||||
void print_allocation_info(size_t device_total, size_t gpu_budget, size_t tile_pixels, int tile_rows)
|
||||
{
|
||||
std::cout
|
||||
<< std::fixed << std::setprecision(1) << " Device pool: " << device_total / (1024.0 * 1024.0) << " MB (initial), "
|
||||
<< gpu_budget / (1024.0 * 1024.0) << " MB (max)\n"
|
||||
<< " Tile size: " << tile_pixels << " pixels (" << tile_rows << " rows x " << image_width << " cols)\n"
|
||||
<< " Pinned host: image=" << image_pixels * sizeof(pixel_t) / (1024.0 * 1024.0)
|
||||
<< " MB, histogram=" << num_bins * sizeof(int) << " bytes\n\n"
|
||||
<< std::defaultfloat << std::setprecision(6);
|
||||
}
|
||||
|
||||
void print_pool_stats(tile_buffers& bufs)
|
||||
{
|
||||
const auto reserved = bufs.device_pool.get().attribute(cuda::memory_pool_attributes::reserved_mem_current);
|
||||
const auto used = bufs.device_pool.get().attribute(cuda::memory_pool_attributes::used_mem_current);
|
||||
std::cout
|
||||
<< std::fixed << std::setprecision(1) << " Device pool: reserved=" << reserved / (1024.0 * 1024.0)
|
||||
<< " MB, used=" << used / (1024.0 * 1024.0) << " MB\n"
|
||||
<< std::defaultfloat << std::setprecision(6);
|
||||
}
|
||||
|
||||
iqr_result compute_iqr(cuda::std::span<const histogram_count_t> hist, size_t total)
|
||||
{
|
||||
auto find_percentile = [&](float pct) {
|
||||
const auto target = static_cast<histogram_count_t>(static_cast<double>(total) * pct);
|
||||
histogram_count_t cumulative{};
|
||||
for (size_t i = 0; i < hist.size(); ++i)
|
||||
{
|
||||
cumulative += hist[i];
|
||||
if (cumulative >= target)
|
||||
{
|
||||
return static_cast<int>(i);
|
||||
}
|
||||
}
|
||||
return static_cast<int>(hist.size()) - 1;
|
||||
};
|
||||
return {find_percentile(0.25f), find_percentile(0.75f)};
|
||||
}
|
||||
|
||||
void print_pass_stats(double ms, long long total_selected, double mean_selected, float global_min, float global_max)
|
||||
{
|
||||
// Note: times measured via cuda::timed_event are approximate GPU-side measurements.
|
||||
std::cout
|
||||
<< std::fixed << std::setprecision(1) << " Pass time: ~" << ms << " ms\n"
|
||||
<< " Pixels above threshold: " << total_selected << " / " << image_pixels << " ("
|
||||
<< 100.0 * total_selected / image_pixels << "%)\n"
|
||||
<< std::setprecision(4) << " Selected range: [" << global_min << ", " << global_max << "], mean=" << mean_selected
|
||||
<< '\n'
|
||||
<< std::defaultfloat << std::setprecision(6);
|
||||
}
|
||||
|
||||
void print_sanity_check(iqr_result orig, iqr_result eq)
|
||||
{
|
||||
const bool ok = eq.width() > orig.width();
|
||||
std::cout
|
||||
<< "=== Sanity check ===\n"
|
||||
<< " Original IQR (25th-75th): [" << orig.p25 << ", " << orig.p75 << "] (span " << orig.width() << ")\n"
|
||||
<< " Equalized IQR (25th-75th): [" << eq.p25 << ", " << eq.p75 << "] (span " << eq.width() << ")\n"
|
||||
<< " Equalization spread distribution: " << (ok ? "YES" : "NO") << "\n\n";
|
||||
}
|
||||
|
||||
void print_summary(int num_tiles, int tile_rows, double pass1_ms, double pass2_ms, bool ok)
|
||||
{
|
||||
std::cout
|
||||
<< "=== Summary ===\n"
|
||||
<< " Image: " << image_width << " x " << image_height << '\n'
|
||||
<< " Tiles: " << num_tiles << " (" << tile_rows << " rows each)\n"
|
||||
<< std::fixed << std::setprecision(1) << " Pass 1 (hist): ~" << pass1_ms << " ms\n"
|
||||
<< " Pass 2 (stats): ~" << pass2_ms << " ms\n"
|
||||
<< " Total pipeline: ~" << pass1_ms + pass2_ms << " ms\n"
|
||||
<< " Result: " << (ok ? "PASSED" : "FAILED") << '\n'
|
||||
<< std::defaultfloat << std::setprecision(6);
|
||||
}
|
||||
|
||||
bool write_bmp(const char* filename, cuda::std::span<const pixel_t> data, int width, int height)
|
||||
{
|
||||
std::ofstream file(filename, std::ios::binary);
|
||||
if (!file)
|
||||
{
|
||||
std::cerr << "Failed to open " << filename << " for writing\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
// BMP rows must be padded to a 4-byte boundary.
|
||||
const int row_stride = (width + 3) & ~3;
|
||||
const int pixel_size = row_stride * height;
|
||||
const int file_size = 54 + 256 * 4 + pixel_size; // header + palette + pixels
|
||||
|
||||
auto put2 = [&](int v) {
|
||||
const char b[2] = {static_cast<char>(v & 0xFF), static_cast<char>((v >> 8) & 0xFF)};
|
||||
file.write(b, 2);
|
||||
};
|
||||
auto put4 = [&](int v) {
|
||||
const char b[4] = {static_cast<char>(v & 0xFF),
|
||||
static_cast<char>((v >> 8) & 0xFF),
|
||||
static_cast<char>((v >> 16) & 0xFF),
|
||||
static_cast<char>((v >> 24) & 0xFF)};
|
||||
file.write(b, 4);
|
||||
};
|
||||
|
||||
// File header (14 bytes).
|
||||
file.put('B');
|
||||
file.put('M');
|
||||
put4(file_size);
|
||||
put4(0); // reserved
|
||||
put4(54 + 256 * 4); // pixel data offset (after header + palette)
|
||||
|
||||
// DIB header (BITMAPINFOHEADER, 40 bytes).
|
||||
put4(40); // header size
|
||||
put4(width);
|
||||
put4(height);
|
||||
put2(1); // color planes
|
||||
put2(8); // bits per pixel (8-bit indexed)
|
||||
put4(0); // no compression
|
||||
put4(pixel_size);
|
||||
put4(2835); // horizontal resolution (72 DPI)
|
||||
put4(2835); // vertical resolution
|
||||
put4(256); // palette entries
|
||||
put4(0); // all colors important
|
||||
|
||||
// Grayscale palette: 256 entries of (B, G, R, 0).
|
||||
for (int i = 0; i < 256; ++i)
|
||||
{
|
||||
const char c = static_cast<char>(i);
|
||||
file.put(c);
|
||||
file.put(c);
|
||||
file.put(c);
|
||||
file.put(0);
|
||||
}
|
||||
|
||||
// Pixel data — BMP stores rows bottom-to-top.
|
||||
const char pad[3] = {0, 0, 0};
|
||||
const int pad_bytes = row_stride - width;
|
||||
for (int y = height - 1; y >= 0; --y)
|
||||
{
|
||||
file.write(reinterpret_cast<const char*>(&data[static_cast<size_t>(y) * width]), width);
|
||||
if (pad_bytes > 0)
|
||||
{
|
||||
file.write(pad, pad_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
if (!file)
|
||||
{
|
||||
std::cerr << "Failed to write " << filename << '\n';
|
||||
return false;
|
||||
}
|
||||
|
||||
std::cout << " Wrote " << filename << " (" << width << " x " << height << ")\n";
|
||||
return true;
|
||||
}
|
||||
63
cccl_upstream/examples/image_pipeline/detail.h
Normal file
63
cccl_upstream/examples/image_pipeline/detail.h
Normal file
@@ -0,0 +1,63 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef DETAIL_H
|
||||
#define DETAIL_H
|
||||
|
||||
/// @file
|
||||
/// Supporting details for the image pipeline example: synthetic image
|
||||
/// generation and printing/output helpers. These are not part of the
|
||||
/// pipeline itself — in a real application, image generation would be
|
||||
/// replaced by actual data loading, and the printing would be replaced
|
||||
/// by your application's logging.
|
||||
|
||||
#include <image_pipeline.h>
|
||||
|
||||
// ── Image generation ─────────────────────────────────────────────────
|
||||
|
||||
/// Generate a synthetic space observation on the GPU, tile by tile,
|
||||
/// and produce a downscaled input preview.
|
||||
/// In a real application this would be replaced by loading actual data.
|
||||
void generate_image(cuda::stream_ref stream, tile_buffers& bufs, int num_tiles, cuda::std::span<pixel_t> host_preview);
|
||||
|
||||
// ── Printing / output helpers ────────────────────────────────────────
|
||||
|
||||
void print_device_info(cuda::device_ref dev, cuda::arch_traits_t traits, size_t total_mem);
|
||||
void print_tile_plan(int tile_rows, int tile_alignment, int num_tiles, size_t budget, size_t total_mem);
|
||||
void print_allocation_info(size_t device_total, size_t gpu_budget, size_t tile_pixels, int tile_rows);
|
||||
void print_pool_stats(tile_buffers& bufs);
|
||||
|
||||
struct iqr_result
|
||||
{
|
||||
int p25, p75;
|
||||
[[nodiscard]] int width() const noexcept
|
||||
{
|
||||
return p75 - p25;
|
||||
}
|
||||
};
|
||||
|
||||
[[nodiscard]] iqr_result compute_iqr(cuda::std::span<const histogram_count_t> hist, size_t total);
|
||||
void print_pass_stats(double ms, long long total_selected, double mean_selected, float global_min, float global_max);
|
||||
void print_sanity_check(iqr_result orig, iqr_result eq);
|
||||
void print_summary(int num_tiles, int tile_rows, double pass1_ms, double pass2_ms, bool ok);
|
||||
[[nodiscard]] bool write_bmp(const char* filename, cuda::std::span<const pixel_t> data, int width, int height);
|
||||
|
||||
/// Downscale a device tile using CUB BlockReduce and copy the result
|
||||
/// to the host preview buffer.
|
||||
void downscale_tile(
|
||||
cuda::stream_ref stream,
|
||||
tile_buffers& bufs,
|
||||
int slot,
|
||||
cuda::std::span<const pixel_t> dev_src,
|
||||
int row_offset,
|
||||
int tile_rows,
|
||||
cuda::std::span<pixel_t> host_preview);
|
||||
|
||||
#endif // DETAIL_H
|
||||
85
cccl_upstream/examples/image_pipeline/image_pipeline.h
Normal file
85
cccl_upstream/examples/image_pipeline/image_pipeline.h
Normal file
@@ -0,0 +1,85 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef IMAGE_PIPELINE_H
|
||||
#define IMAGE_PIPELINE_H
|
||||
|
||||
#include <cuda/buffer>
|
||||
#include <cuda/devices>
|
||||
#include <cuda/memory_resource>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/cstdint>
|
||||
#include <cuda/std/limits>
|
||||
#include <cuda/std/span>
|
||||
#include <cuda/stream>
|
||||
|
||||
// ── Constants ────────────────────────────────────────────────────────
|
||||
|
||||
/// Pixel type: 8-bit grayscale.
|
||||
using pixel_t = uint8_t;
|
||||
|
||||
/// Full-image histogram count type.
|
||||
using histogram_count_t = long long;
|
||||
|
||||
/// Image dimensions. 65K x 65K (~4 GB raw). Large enough that
|
||||
/// the working set won't fit in most GPUs, forcing real tiling.
|
||||
/// Note: the full image is held in pinned host memory (~4 GB).
|
||||
/// Ensure the system has at least 8 GB of RAM.
|
||||
inline constexpr int image_width = 65536;
|
||||
inline constexpr int image_height = 65536;
|
||||
inline constexpr size_t image_pixels = static_cast<size_t>(image_width) * image_height;
|
||||
|
||||
/// Preview downscale factor. The output preview images are
|
||||
/// (image_width / preview_scale) x (image_height / preview_scale).
|
||||
inline constexpr int preview_scale = 64;
|
||||
|
||||
/// Histogram bins (one per possible grayscale value).
|
||||
inline constexpr int num_bins = cuda::std::numeric_limits<pixel_t>::max() + 1;
|
||||
inline constexpr int num_levels = num_bins + 1; // CUB needs num_levels = num_bins + 1
|
||||
|
||||
// ── Shared data structures ───────────────────────────────────────────
|
||||
|
||||
/// Device selection and tile sizing results.
|
||||
struct device_plan
|
||||
{
|
||||
cuda::device_ref device;
|
||||
int tile_rows;
|
||||
int num_tiles;
|
||||
size_t gpu_budget;
|
||||
};
|
||||
|
||||
/// All working memory for the pipeline.
|
||||
struct tile_buffers
|
||||
{
|
||||
cuda::device_buffer<pixel_t> dev_tile[2]; // double-buffered H2D
|
||||
cuda::device_buffer<float> dev_float_tile[2]; // normalized float tile
|
||||
cuda::device_buffer<int> dev_histogram[2]; // per-tile histogram
|
||||
cuda::device_buffer<float4> dev_tile_stats; // per-tile reduction: {count, min, max, sum}
|
||||
cuda::device_buffer<pixel_t> dev_lut; // equalization LUT
|
||||
cuda::device_buffer<pixel_t> dev_equalized[2]; // equalized tile
|
||||
cuda::host_buffer<pixel_t> host_image; // full image in pinned memory
|
||||
cuda::host_buffer<int> host_tile_histograms; // per-tile histograms
|
||||
cuda::host_buffer<float4> host_tile_stats; // per-tile stats readback
|
||||
cuda::device_buffer<pixel_t> dev_preview[2]; // downscaled preview tile
|
||||
|
||||
cuda::mr::shared_resource<cuda::device_memory_pool> device_pool;
|
||||
size_t tile_pixels;
|
||||
};
|
||||
|
||||
/// Per-tile processing statistics.
|
||||
struct tile_stats
|
||||
{
|
||||
float min_val;
|
||||
float max_val;
|
||||
float sum;
|
||||
long long num_selected;
|
||||
};
|
||||
|
||||
#endif // IMAGE_PIPELINE_H
|
||||
616
cccl_upstream/examples/image_pipeline/main.cu
Normal file
616
cccl_upstream/examples/image_pipeline/main.cu
Normal file
@@ -0,0 +1,616 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
/**
|
||||
* Image processing pipeline — all runtime API usage in one file.
|
||||
*
|
||||
* This file contains the complete pipeline:
|
||||
* - Device selection and tile sizing
|
||||
* - Memory pool creation and buffer allocation
|
||||
* - Tile upload/download with copy_bytes and fill_bytes
|
||||
* - CUB-based processing (histogram, equalization, thresholding, reduction)
|
||||
* - GPU downscale with CUB BlockReduce
|
||||
* - Double-buffered two-stream orchestration
|
||||
*
|
||||
* Supporting details (image generation, printing) are in detail.cu.
|
||||
*/
|
||||
|
||||
#include <cub/block/block_reduce.cuh>
|
||||
#include <cub/device/device_histogram.cuh>
|
||||
#include <cub/device/device_reduce.cuh>
|
||||
#include <cub/device/device_transform.cuh>
|
||||
|
||||
#include <cuda/algorithm>
|
||||
#include <cuda/buffer>
|
||||
#include <cuda/cmath>
|
||||
#include <cuda/devices>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/memory_pool>
|
||||
#include <cuda/memory_resource>
|
||||
#include <cuda/std/__exception/cuda_error.h>
|
||||
#include <cuda/std/algorithm>
|
||||
#include <cuda/std/array>
|
||||
#include <cuda/std/execution>
|
||||
#include <cuda/std/functional>
|
||||
#include <cuda/std/limits>
|
||||
#include <cuda/std/span>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <exception>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
|
||||
#include <cuda_runtime_api.h>
|
||||
#include <detail.h>
|
||||
#include <image_pipeline.h>
|
||||
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
// Device selection
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
|
||||
static device_plan select_device_and_plan()
|
||||
{
|
||||
std::cout << "=== Device selection ===\n";
|
||||
|
||||
cuda::device_ref best = cuda::devices[0];
|
||||
size_t best_mem = 0;
|
||||
|
||||
for (auto dev : cuda::devices)
|
||||
{
|
||||
const size_t total_bytes = dev.attribute(cuda::device_attributes::total_global_memory);
|
||||
const int sms = dev.attribute(cuda::device_attributes::multiprocessor_count);
|
||||
const auto name = dev.name();
|
||||
std::cout << " [" << dev.get() << "] ";
|
||||
std::cout.write(name.data(), static_cast<std::streamsize>(name.size()));
|
||||
std::cout
|
||||
<< " " << std::setw(3) << sms << " SMs " << std::fixed << std::setprecision(0)
|
||||
<< total_bytes / (1024.0 * 1024.0) << " MB\n"
|
||||
<< std::defaultfloat << std::setprecision(6);
|
||||
|
||||
if (total_bytes > best_mem)
|
||||
{
|
||||
best = dev;
|
||||
best_mem = total_bytes;
|
||||
}
|
||||
}
|
||||
|
||||
const auto cc = best.attribute(cuda::device_attributes::compute_capability);
|
||||
const auto traits = cuda::arch_traits_for(cc);
|
||||
print_device_info(best, traits, best_mem);
|
||||
|
||||
// Budget 60% of total GPU memory for the per-tile working set.
|
||||
const size_t budget = static_cast<size_t>(best_mem * 0.60);
|
||||
const size_t overhead = 128 * 1024 * 1024;
|
||||
const size_t bytes_per_pixel = 4 * sizeof(pixel_t) + 2 * sizeof(float);
|
||||
const size_t usable_budget = (budget > overhead) ? (budget - overhead) : 0;
|
||||
const size_t budget_rows = usable_budget / bytes_per_pixel / image_width;
|
||||
|
||||
constexpr int tile_alignment = preview_scale;
|
||||
const size_t max_launch_rows =
|
||||
(static_cast<size_t>(cuda::std::numeric_limits<int>::max()) / image_width / tile_alignment) * tile_alignment;
|
||||
const size_t max_tile_rows = cuda::std::min(static_cast<size_t>(image_height), max_launch_rows);
|
||||
const size_t aligned_tile_rows = (budget_rows / tile_alignment) * tile_alignment;
|
||||
const auto clamped_tile_rows =
|
||||
cuda::std::clamp(aligned_tile_rows, static_cast<size_t>(tile_alignment), max_tile_rows);
|
||||
const int tile_rows = static_cast<int>(clamped_tile_rows);
|
||||
const int num_tiles = cuda::ceil_div(image_height, tile_rows);
|
||||
|
||||
print_tile_plan(tile_rows, tile_alignment, num_tiles, budget, best_mem);
|
||||
return {best, tile_rows, num_tiles, budget};
|
||||
}
|
||||
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
// Buffer allocation
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
|
||||
static tile_buffers
|
||||
allocate_tile_buffers(cuda::stream_ref stream, cuda::device_ref device, int tile_rows, size_t gpu_budget, int num_tiles)
|
||||
{
|
||||
std::cout << "=== Tile buffer allocation ===\n";
|
||||
const size_t tile_pixels = static_cast<size_t>(tile_rows) * image_width;
|
||||
const size_t preview_px = (tile_rows / preview_scale) * (image_width / preview_scale);
|
||||
|
||||
// Single pool for our buffers and CUB temporaries.
|
||||
const size_t device_total =
|
||||
2 * tile_pixels * sizeof(pixel_t) // double-buffered pixel tiles
|
||||
+ 2 * tile_pixels * sizeof(pixel_t) // double-buffered equalized tiles
|
||||
+ 2 * tile_pixels * sizeof(float) // double-buffered normalized float tiles
|
||||
+ sizeof(float4) // reduction output
|
||||
+ num_bins * sizeof(pixel_t) // equalization LUT
|
||||
+ 2 * num_bins * sizeof(int) // double-buffered histograms
|
||||
+ 2 * preview_px * sizeof(pixel_t); // double-buffered preview tiles
|
||||
|
||||
cuda::memory_pool_properties props{};
|
||||
props.initial_pool_size = device_total;
|
||||
props.max_pool_size = gpu_budget;
|
||||
|
||||
auto device_pool = cuda::mr::shared_resource<cuda::device_memory_pool>(
|
||||
cuda::std::in_place_type<cuda::device_memory_pool>, device, props);
|
||||
|
||||
// Device buffers — no_init since kernels/copies write before reading.
|
||||
auto dev_tile_0 = cuda::make_buffer<pixel_t>(stream, device_pool, tile_pixels, cuda::no_init);
|
||||
auto dev_tile_1 = cuda::make_buffer<pixel_t>(stream, device_pool, tile_pixels, cuda::no_init);
|
||||
auto dev_float_0 = cuda::make_buffer<float>(stream, device_pool, tile_pixels, cuda::no_init);
|
||||
auto dev_float_1 = cuda::make_buffer<float>(stream, device_pool, tile_pixels, cuda::no_init);
|
||||
auto dev_hist_0 = cuda::make_buffer<int>(stream, device_pool, num_bins, cuda::no_init);
|
||||
auto dev_hist_1 = cuda::make_buffer<int>(stream, device_pool, num_bins, cuda::no_init);
|
||||
auto dev_tile_stats = cuda::make_buffer<float4>(stream, device_pool, num_tiles, cuda::no_init);
|
||||
auto dev_lut = cuda::make_buffer<pixel_t>(stream, device_pool, num_bins, cuda::no_init);
|
||||
auto dev_equalized_0 = cuda::make_buffer<pixel_t>(stream, device_pool, tile_pixels, cuda::no_init);
|
||||
auto dev_equalized_1 = cuda::make_buffer<pixel_t>(stream, device_pool, tile_pixels, cuda::no_init);
|
||||
auto dev_preview_0 = cuda::make_buffer<pixel_t>(stream, device_pool, preview_px, cuda::no_init);
|
||||
auto dev_preview_1 = cuda::make_buffer<pixel_t>(stream, device_pool, preview_px, cuda::no_init);
|
||||
|
||||
// Pinned host buffers — make_pinned_buffer uses the default pinned pool.
|
||||
auto host_image = cuda::make_pinned_buffer<pixel_t>(stream, image_pixels, pixel_t{0});
|
||||
auto host_tile_hists = cuda::make_pinned_buffer<int>(stream, static_cast<size_t>(num_tiles) * num_bins, int{0});
|
||||
auto host_tile_stats = cuda::make_pinned_buffer<float4>(stream, num_tiles, float4{0, 0, 0, 0});
|
||||
|
||||
print_allocation_info(device_total, gpu_budget, tile_pixels, tile_rows);
|
||||
|
||||
return {
|
||||
{cuda::std::move(dev_tile_0), cuda::std::move(dev_tile_1)},
|
||||
{cuda::std::move(dev_float_0), cuda::std::move(dev_float_1)},
|
||||
{cuda::std::move(dev_hist_0), cuda::std::move(dev_hist_1)},
|
||||
cuda::std::move(dev_tile_stats),
|
||||
cuda::std::move(dev_lut),
|
||||
{cuda::std::move(dev_equalized_0), cuda::std::move(dev_equalized_1)},
|
||||
cuda::std::move(host_image),
|
||||
cuda::std::move(host_tile_hists),
|
||||
cuda::std::move(host_tile_stats),
|
||||
{cuda::std::move(dev_preview_0), cuda::std::move(dev_preview_1)},
|
||||
device_pool,
|
||||
tile_pixels,
|
||||
};
|
||||
}
|
||||
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
// Tile transfer helpers
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
|
||||
static size_t upload_tile(cuda::stream_ref stream, tile_buffers& bufs, int slot, int tile_idx, int tile_rows)
|
||||
{
|
||||
const size_t offset = static_cast<size_t>(tile_idx) * bufs.tile_pixels;
|
||||
const size_t count = cuda::std::min(bufs.tile_pixels, image_pixels - offset);
|
||||
|
||||
cuda::copy_configuration config{};
|
||||
config.src_access_order = cuda::source_access_order::stream;
|
||||
cuda::copy_bytes(stream, bufs.host_image.subspan(offset, count), bufs.dev_tile[slot].first(count), config);
|
||||
cuda::fill_bytes(stream, bufs.dev_histogram[slot], uint8_t{0});
|
||||
return count;
|
||||
}
|
||||
|
||||
static void download_tile_histogram(cuda::stream_ref stream, tile_buffers& bufs, int slot, int tile_idx)
|
||||
{
|
||||
const size_t offset = static_cast<size_t>(tile_idx) * num_bins;
|
||||
cuda::copy_bytes(stream, bufs.dev_histogram[slot], bufs.host_tile_histograms.subspan(offset, num_bins));
|
||||
}
|
||||
|
||||
static void accumulate_histograms(tile_buffers& bufs, int num_tiles, cuda::std::span<histogram_count_t> result)
|
||||
{
|
||||
for (int i = 0; i < num_bins; ++i)
|
||||
{
|
||||
result[i] = 0;
|
||||
}
|
||||
for (int t = 0; t < num_tiles; ++t)
|
||||
{
|
||||
const size_t offset = static_cast<size_t>(t) * num_bins;
|
||||
for (int i = 0; i < num_bins; ++i)
|
||||
{
|
||||
result[i] += bufs.host_tile_histograms.get_unsynchronized(offset + i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void upload_lut(cuda::stream_ref stream, tile_buffers& bufs, cuda::std::span<const pixel_t> host_lut)
|
||||
{
|
||||
cuda::copy_bytes(stream, host_lut, bufs.dev_lut);
|
||||
}
|
||||
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
// CUB-based processing
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
|
||||
static auto make_cub_env(cuda::stream_ref stream, cuda::mr::shared_resource<cuda::device_memory_pool>& pool)
|
||||
{
|
||||
const auto mr_prop = cuda::std::execution::prop{cuda::mr::get_memory_resource_t{}, pool};
|
||||
return cuda::std::execution::env{stream, mr_prop};
|
||||
}
|
||||
|
||||
static void check_cub(cudaError_t err, const char* msg)
|
||||
{
|
||||
if (err != cudaSuccess)
|
||||
{
|
||||
throw cuda::cuda_error(err, msg, "CUB");
|
||||
}
|
||||
}
|
||||
|
||||
// ── Processing functions ─────────────────────────────────────────────
|
||||
|
||||
static void compute_histogram(cuda::stream_ref stream, tile_buffers& bufs, int slot, size_t tile_pixel_count)
|
||||
{
|
||||
auto env = make_cub_env(stream, bufs.device_pool);
|
||||
check_cub(
|
||||
cub::DeviceHistogram::HistogramEven(
|
||||
bufs.dev_tile[slot].first(tile_pixel_count).data(),
|
||||
bufs.dev_histogram[slot].data(),
|
||||
num_levels,
|
||||
0,
|
||||
num_bins,
|
||||
static_cast<int>(tile_pixel_count),
|
||||
env),
|
||||
"HistogramEven (pass 1)");
|
||||
}
|
||||
|
||||
static void process_tile(
|
||||
cuda::stream_ref stream, tile_buffers& bufs, int slot, int tile_idx, size_t tile_pixel_count, float threshold)
|
||||
{
|
||||
const int n = static_cast<int>(tile_pixel_count);
|
||||
auto pixel_data = bufs.dev_tile[slot].first(tile_pixel_count).data();
|
||||
auto eq_data = bufs.dev_equalized[slot].data();
|
||||
auto float_data = bufs.dev_float_tile[slot].data();
|
||||
auto env = make_cub_env(stream, bufs.device_pool);
|
||||
|
||||
// Equalize: apply the LUT to remap pixel intensities.
|
||||
auto lut_span = bufs.dev_lut.first(num_bins);
|
||||
auto equalize = [lut_span] __device__(pixel_t p) -> pixel_t {
|
||||
return lut_span[p];
|
||||
};
|
||||
check_cub(cub::DeviceTransform::Transform(pixel_data, eq_data, n, equalize, env), "Transform (equalize)");
|
||||
|
||||
// Normalize: convert uint8 pixels to [0, 1] floats.
|
||||
auto normalize = [] __device__(pixel_t p) -> float {
|
||||
constexpr float pixel_max = cuda::std::numeric_limits<pixel_t>::max();
|
||||
return static_cast<float>(p) / pixel_max;
|
||||
};
|
||||
check_cub(cub::DeviceTransform::Transform(eq_data, float_data, n, normalize, env), "Transform (normalize)");
|
||||
|
||||
check_cub(
|
||||
cub::DeviceHistogram::HistogramEven(eq_data, bufs.dev_histogram[slot].data(), num_levels, 0, num_bins, n, env),
|
||||
"HistogramEven (pass 2)");
|
||||
|
||||
// Combined threshold + count/min/max/sum in a single pass via float4: x=count, y=min, z=max, w=sum.
|
||||
// Each tile writes to its own output index — no sync needed between tiles.
|
||||
constexpr float flt_max = cuda::std::numeric_limits<float>::max();
|
||||
constexpr float flt_low = cuda::std::numeric_limits<float>::lowest();
|
||||
const float4 identity{0.0f, flt_max, flt_low, 0.0f};
|
||||
|
||||
auto threshold_stats = [threshold] __device__(float v) -> float4 {
|
||||
if (v > threshold)
|
||||
{
|
||||
return {1.0f, v, v, v}; // count=1, min=v, max=v, sum=v
|
||||
}
|
||||
return {0.0f, flt_max, flt_low, 0.0f};
|
||||
};
|
||||
|
||||
auto stats_reduce = [] __device__(float4 a, float4 b) -> float4 {
|
||||
return {a.x + b.x, cuda::std::min(a.y, b.y), cuda::std::max(a.z, b.z), a.w + b.w};
|
||||
};
|
||||
|
||||
check_cub(cub::DeviceReduce::TransformReduce(
|
||||
float_data, bufs.dev_tile_stats.data() + tile_idx, n, stats_reduce, threshold_stats, identity, env),
|
||||
"TransformReduce");
|
||||
|
||||
// D2H copy into this tile's slot — no sync, read after all tiles finish.
|
||||
cuda::copy_bytes(stream, bufs.dev_tile_stats.subspan(tile_idx, 1), bufs.host_tile_stats.subspan(tile_idx, 1));
|
||||
}
|
||||
|
||||
/// Accumulate per-tile stats into a single result after all tiles finish.
|
||||
static tile_stats accumulate_tile_stats(tile_buffers& bufs, int num_tiles)
|
||||
{
|
||||
tile_stats result{};
|
||||
result.min_val = cuda::std::numeric_limits<float>::max();
|
||||
result.max_val = cuda::std::numeric_limits<float>::lowest();
|
||||
|
||||
for (int t = 0; t < num_tiles; ++t)
|
||||
{
|
||||
const auto s = bufs.host_tile_stats.get_unsynchronized(t);
|
||||
result.num_selected += static_cast<long long>(s.x);
|
||||
result.min_val = cuda::std::min(result.min_val, s.y);
|
||||
result.max_val = cuda::std::max(result.max_val, s.z);
|
||||
result.sum += s.w;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Host-side algorithms ─────────────────────────────────────────────
|
||||
|
||||
static float compute_otsu_threshold(cuda::std::span<const histogram_count_t> histogram, size_t total_pixels)
|
||||
{
|
||||
double total_sum = 0;
|
||||
for (int i = 0; i < num_bins; ++i)
|
||||
{
|
||||
total_sum += static_cast<double>(i) * histogram[i];
|
||||
}
|
||||
double sum_bg = 0, weight_bg = 0, max_var = 0;
|
||||
int best_t = 0;
|
||||
for (int t = 0; t < num_bins; ++t)
|
||||
{
|
||||
weight_bg += histogram[t];
|
||||
if (weight_bg == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
const double weight_fg = static_cast<double>(total_pixels) - weight_bg;
|
||||
if (weight_fg == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
sum_bg += static_cast<double>(t) * histogram[t];
|
||||
const double mean_bg = sum_bg / weight_bg;
|
||||
const double mean_fg = (total_sum - sum_bg) / weight_fg;
|
||||
const double var = weight_bg * weight_fg * (mean_bg - mean_fg) * (mean_bg - mean_fg);
|
||||
if (var > max_var)
|
||||
{
|
||||
max_var = var;
|
||||
best_t = t;
|
||||
}
|
||||
}
|
||||
return static_cast<float>(best_t) / cuda::std::numeric_limits<pixel_t>::max();
|
||||
}
|
||||
|
||||
static void build_equalization_lut(
|
||||
cuda::std::span<const histogram_count_t> histogram, size_t total_pixels, cuda::std::span<pixel_t> lut_out)
|
||||
{
|
||||
constexpr double max_val = cuda::std::numeric_limits<pixel_t>::max();
|
||||
const double scale = max_val / static_cast<double>(total_pixels);
|
||||
double cdf = 0;
|
||||
for (int i = 0; i < num_bins; ++i)
|
||||
{
|
||||
cdf += histogram[i];
|
||||
lut_out[i] = static_cast<pixel_t>(cuda::std::min(cdf * scale, max_val));
|
||||
}
|
||||
}
|
||||
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
// Downscale kernel
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
|
||||
// Each block produces one output pixel by box-averaging a scale×scale
|
||||
// source block. Threads cooperatively load source pixels and sum
|
||||
// them locally, then cub::BlockReduce merges the partial sums.
|
||||
//
|
||||
// The block size is extracted from the launch config at compile time
|
||||
// via cuda::gpu_thread.count(cuda::block, config), which is then used
|
||||
// as the template parameter for cub::BlockReduce.
|
||||
|
||||
struct downscale_kernel
|
||||
{
|
||||
template <typename Config>
|
||||
__device__ void
|
||||
operator()(Config config, cuda::std::span<const pixel_t> src, cuda::std::span<pixel_t> dst, int src_width, int scale)
|
||||
{
|
||||
constexpr int block_size = cuda::gpu_thread.count(cuda::block, config);
|
||||
const int out_idx = blockIdx.x;
|
||||
if (out_idx >= static_cast<int>(dst.size()))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const int dst_width = src_width / scale;
|
||||
const int px = out_idx % dst_width;
|
||||
const int py = out_idx / dst_width;
|
||||
const int total_elems = scale * scale;
|
||||
|
||||
// Each thread sums its share of the scale×scale source block.
|
||||
int local_sum = 0;
|
||||
const int tid = threadIdx.x;
|
||||
for (int i = tid; i < total_elems; i += block_size)
|
||||
{
|
||||
const int dy = i / scale;
|
||||
const int dx = i % scale;
|
||||
local_sum += src[static_cast<size_t>(py * scale + dy) * src_width + (px * scale + dx)];
|
||||
}
|
||||
|
||||
// cub::BlockReduce sums the per-thread partial sums into a single
|
||||
// block-wide total. Without CUB, this would be a manual shared-
|
||||
// memory tree reduction:
|
||||
//
|
||||
// __shared__ int smem[block_size];
|
||||
// smem[tid] = local_sum;
|
||||
// __syncthreads();
|
||||
// for (int s = block_size / 2; s > 0; s >>= 1)
|
||||
// {
|
||||
// if (tid < s) smem[tid] += smem[tid + s];
|
||||
// __syncthreads();
|
||||
// }
|
||||
// int block_sum = smem[0];
|
||||
using BlockReduceT = cub::BlockReduce<int, block_size>;
|
||||
__shared__ typename BlockReduceT::TempStorage temp_storage;
|
||||
const int block_sum = BlockReduceT(temp_storage).Sum(local_sum);
|
||||
|
||||
if (tid == 0)
|
||||
{
|
||||
dst[out_idx] = static_cast<pixel_t>(block_sum / total_elems);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void downscale_tile(
|
||||
cuda::stream_ref stream,
|
||||
tile_buffers& bufs,
|
||||
int slot,
|
||||
cuda::std::span<const pixel_t> dev_src,
|
||||
int row_offset,
|
||||
int tile_rows,
|
||||
cuda::std::span<pixel_t> host_preview)
|
||||
{
|
||||
const int dst_rows = tile_rows / preview_scale;
|
||||
const int dst_cols = image_width / preview_scale;
|
||||
const int dst_pixels = dst_rows * dst_cols;
|
||||
if (dst_pixels == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
constexpr int block_size = 256;
|
||||
const auto config = cuda::make_config(cuda::block_dims<block_size>(), cuda::grid_dims(dst_pixels));
|
||||
|
||||
cuda::launch(
|
||||
stream,
|
||||
config,
|
||||
downscale_kernel{},
|
||||
dev_src,
|
||||
bufs.dev_preview[slot].first(static_cast<size_t>(dst_pixels)),
|
||||
image_width,
|
||||
preview_scale);
|
||||
|
||||
const int preview_row_offset = row_offset / preview_scale;
|
||||
cuda::copy_bytes(
|
||||
stream,
|
||||
bufs.dev_preview[slot].first(static_cast<size_t>(dst_pixels)),
|
||||
host_preview.subspan(static_cast<size_t>(preview_row_offset) * dst_cols, static_cast<size_t>(dst_pixels)));
|
||||
}
|
||||
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
// Main
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
|
||||
int main()
|
||||
try
|
||||
{
|
||||
// ── 1. Device selection and tile sizing ────────────────────────────
|
||||
const auto plan = select_device_and_plan();
|
||||
|
||||
// ── 2. Allocate all buffers ────────────────────────────────────────
|
||||
// Two streams for double-buffered tile processing. stream_a also
|
||||
// handles setup work (allocation, LUT upload, etc.) between passes.
|
||||
cuda::stream stream_a{plan.device};
|
||||
cuda::stream stream_b{plan.device};
|
||||
cuda::stream_ref streams[2] = {stream_a, stream_b};
|
||||
|
||||
auto bufs = allocate_tile_buffers(stream_a, plan.device, plan.tile_rows, plan.gpu_budget, plan.num_tiles);
|
||||
|
||||
// ── 3. Generate image and downscale input preview ──────────────────
|
||||
const int pw = image_width / preview_scale;
|
||||
const int ph = image_height / preview_scale;
|
||||
auto host_input_preview = cuda::make_pinned_buffer<pixel_t>(stream_a, static_cast<size_t>(pw) * ph, pixel_t{0});
|
||||
generate_image(stream_a, bufs, plan.num_tiles, host_input_preview.subspan(0));
|
||||
bool outputs_ok = write_bmp("input_preview.bmp", host_input_preview.subspan(0), pw, ph);
|
||||
|
||||
// ── 4. Pass 1: histogram (double-buffered) ─────────────────────────
|
||||
// stream_a: [upload tile 0] [histogram 0] [download 0] [tile 2] ...
|
||||
// stream_b: [upload tile 1] [histogram 1] [download 1] ...
|
||||
std::cout << "=== Pass 1: histogram ===\n";
|
||||
|
||||
cuda::timed_event pass1_start{stream_a};
|
||||
|
||||
for (int t = 0; t < plan.num_tiles; ++t)
|
||||
{
|
||||
const int slot = t % 2;
|
||||
const size_t count = upload_tile(streams[slot], bufs, slot, t, plan.tile_rows);
|
||||
compute_histogram(streams[slot], bufs, slot, count);
|
||||
download_tile_histogram(streams[slot], bufs, slot, t);
|
||||
}
|
||||
|
||||
stream_a.wait(stream_b);
|
||||
cuda::timed_event pass1_end{stream_a};
|
||||
stream_a.sync();
|
||||
const double pass1_ms = (pass1_end - pass1_start).count() / 1e6;
|
||||
std::cout << std::fixed << std::setprecision(1) << " Histogram pass: " << pass1_ms << " ms\n"
|
||||
<< std::defaultfloat << std::setprecision(6);
|
||||
|
||||
// ── 5. Otsu threshold + equalization LUT ───────────────────────────
|
||||
cuda::std::array<histogram_count_t, num_bins> original_hist{};
|
||||
cuda::std::span global_hist_span{original_hist};
|
||||
accumulate_histograms(bufs, plan.num_tiles, global_hist_span);
|
||||
|
||||
const float otsu = compute_otsu_threshold(global_hist_span, image_pixels);
|
||||
std::cout
|
||||
<< std::fixed << std::setprecision(4) << " Otsu threshold: " << otsu << " (" << static_cast<int>(otsu * 255)
|
||||
<< " / 255)\n"
|
||||
<< std::defaultfloat << std::setprecision(6);
|
||||
|
||||
pixel_t host_lut[num_bins];
|
||||
build_equalization_lut(global_hist_span, image_pixels, cuda::std::span<pixel_t>(host_lut, num_bins));
|
||||
|
||||
// Construct a pinned buffer from the host LUT array — the buffer
|
||||
// copies the data in stream order, no manual sync needed.
|
||||
auto pinned_lut = cuda::make_pinned_buffer<pixel_t>(stream_a, host_lut, host_lut + num_bins);
|
||||
upload_lut(stream_a, bufs, pinned_lut.subspan(0));
|
||||
std::cout << " Equalization LUT uploaded\n\n";
|
||||
|
||||
cuda::fill_bytes(stream_a, bufs.host_tile_histograms, uint8_t{0});
|
||||
auto host_eq_preview = cuda::make_pinned_buffer<pixel_t>(stream_a, static_cast<size_t>(pw) * ph, pixel_t{0});
|
||||
|
||||
// stream_b must wait for the setup work on stream_a before starting pass 2.
|
||||
stream_b.wait(stream_a);
|
||||
|
||||
// ── 6. Pass 2: equalize + threshold + stats + preview ──────────────
|
||||
std::cout << "=== Pass 2: equalize + threshold + statistics ===\n";
|
||||
cuda::timed_event pass2_start{stream_a};
|
||||
|
||||
for (int t = 0; t < plan.num_tiles; ++t)
|
||||
{
|
||||
const int slot = t % 2;
|
||||
const size_t count = upload_tile(streams[slot], bufs, slot, t, plan.tile_rows);
|
||||
process_tile(streams[slot], bufs, slot, t, count, otsu);
|
||||
const int tile_rows = static_cast<int>(count / image_width);
|
||||
const int row_offset = t * static_cast<int>(bufs.tile_pixels / image_width);
|
||||
|
||||
downscale_tile(
|
||||
streams[slot],
|
||||
bufs,
|
||||
slot,
|
||||
bufs.dev_equalized[slot].first(count),
|
||||
row_offset,
|
||||
tile_rows,
|
||||
host_eq_preview.subspan(0));
|
||||
download_tile_histogram(streams[slot], bufs, slot, t);
|
||||
}
|
||||
|
||||
// Single sync after all tiles — stats, histograms, and preview are on host.
|
||||
stream_a.wait(stream_b);
|
||||
cuda::timed_event pass2_end{stream_a};
|
||||
stream_a.sync();
|
||||
const double pass2_ms = (pass2_end - pass2_start).count() / 1e6;
|
||||
|
||||
const auto stats = accumulate_tile_stats(bufs, plan.num_tiles);
|
||||
const double mean_selected = (stats.num_selected > 0) ? static_cast<double>(stats.sum) / stats.num_selected : 0.0;
|
||||
|
||||
print_pass_stats(pass2_ms, stats.num_selected, mean_selected, stats.min_val, stats.max_val);
|
||||
print_pool_stats(bufs);
|
||||
|
||||
// ── 7. Write equalized preview ─────────────────────────────────────
|
||||
outputs_ok = write_bmp("equalized_preview.bmp", host_eq_preview.subspan(0), pw, ph) && outputs_ok;
|
||||
if (!outputs_ok)
|
||||
{
|
||||
std::cerr << "One or more preview BMP files were not written.\n";
|
||||
}
|
||||
std::cout << '\n';
|
||||
|
||||
// ── 8. Sanity check ────────────────────────────────────────────────
|
||||
const auto orig_iqr = compute_iqr(cuda::std::span{original_hist}, image_pixels);
|
||||
|
||||
cuda::std::array<histogram_count_t, num_bins> equalized_hist{};
|
||||
accumulate_histograms(bufs, plan.num_tiles, cuda::std::span{equalized_hist});
|
||||
const auto eq_iqr = compute_iqr(cuda::std::span{equalized_hist}, image_pixels);
|
||||
|
||||
print_sanity_check(orig_iqr, eq_iqr);
|
||||
|
||||
const bool ok = outputs_ok && eq_iqr.width() > orig_iqr.width();
|
||||
print_summary(plan.num_tiles, plan.tile_rows, pass1_ms, pass2_ms, ok);
|
||||
return ok ? 0 : 1;
|
||||
}
|
||||
catch (const cuda::cuda_error& e)
|
||||
{
|
||||
std::cerr << "CUDA error: " << e.what() << '\n';
|
||||
return 1;
|
||||
}
|
||||
catch (const std::exception& e)
|
||||
{
|
||||
std::cerr << "Error: " << e.what() << '\n';
|
||||
return 1;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
std::cerr << "An unknown error was encountered\n";
|
||||
return 1;
|
||||
}
|
||||
Reference in New Issue
Block a user