feat(cccl): integrate missing CCCL directories — python/, ci/, .agent/, docs/, test/

Sparse-checkout from NVIDIA/cccl main branch to complete cccl_upstream: Added: - python/cuda_cccl/ (226 files) — Python bindings for device-level algorithms Critical for muh toolchain: cuda.compute.reduce_into, scan, radix_sort, etc. Includes 204 .py files with full test coverage for all 27 algorithms - ci/ (163 files) — Build/test infrastructure build_cub.sh, test_cub.sh, build_and_test_targets.sh, matrix.yaml Directly maps to our [INFRA-CI] and [INFRA-BUILD] items - .agent/skills/ (7 files) — NVIDIA's own agent skills for CCCL cccl-style/SKILL.md, cccl-test/SKILL.md, sass-diff/SKILL.md - docs/ (491 files) — Official CCCL documentation CI references, CMake guides, Python compute docs, libcudacxx PTX docs - test/ (12 files) — Top-level integration tests (cuda_smoke, stdpar) - Root configs: .clang-format, .clang-tidy, CONTRIBUTING.md, pyproject.toml - CLAUDE.md symlink → AGENTS.md (NVIDIA's standard) cccl_upstream now mirrors full NVIDIA/cccl structure: Before: 42M (cub + thrust + libcudacxx + cudax + c + examples + benchmarks) After: 53M (+python +ci +docs +.agent +test +configs) This completes the CCCL base needed for: - [muh-bench] items: ci/util/build_and_test_targets.sh for targeted builds - [CCCL-verify] items: python/cuda_cccl/tests/ as reference implementations - [CCCL-test] items: ci/test_cub.sh, ci/test_thrust.sh - Agent workflow: .agent/skills/ for consistent style and test patterns
2026-08-07 02:34:33 +00:00
parent 3f97dca7ad
commit 2a7ca101d7
908 changed files with 121615 additions and 0 deletions
--- a/cccl_upstream/ci/rapids/cuda13.3-conda/devcontainer.json
+++ b/cccl_upstream/ci/rapids/cuda13.3-conda/devcontainer.json
@@ -0,0 +1,124 @@
+{
+  "image": "rapidsai/devcontainers:26.10-cpp-mambaforge",
+  "runArgs": [
+    "--init",
+    "--rm",
+    "--name",
+    "${localEnv:USER:anon}-${localWorkspaceFolderBasename}-rapids-26.10-cuda13.3-conda",
+    "--ulimit",
+    "nofile=500000"
+  ],
+  "hostRequirements": {
+    "gpu": "optional"
+  },
+  "containerEnv": {
+    "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
+    "CI": "${localEnv:CI}",
+    "CUDA_VERSION": "13.3",
+    "CUDAARCHS": "75-real",
+    "DEFAULT_CONDA_ENV": "rapids",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "true",
+    "HISTFILE": "/home/coder/.cache/._bash_history",
+    "INFER_NUM_DEVICE_ARCHITECTURES": "1",
+    "LIBCUDF_KERNEL_CACHE_PATH": "/home/coder/cudf/cpp/build/latest/jitify_cache",
+    "MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL": "20",
+    "PYTHON_PACKAGE_MANAGER": "conda",
+    "PYTHONDONTWRITEBYTECODE": "1",
+    "PYTHONSAFEPATH": "1",
+    "PYTHONUNBUFFERED": "1",
+    "RAPIDS_cmake_GIT_REPO": "${localEnv:RAPIDS_cmake_GIT_REPO}",
+    "RAPIDS_cudf_GIT_REPO": "${localEnv:RAPIDS_cudf_GIT_REPO}",
+    "RAPIDS_cugraph_GIT_REPO": "${localEnv:RAPIDS_cugraph_GIT_REPO}",
+    "RAPIDS_cugraph_gnn_GIT_REPO": "${localEnv:RAPIDS_cugraph_gnn_GIT_REPO}",
+    "RAPIDS_cuml_GIT_REPO": "${localEnv:RAPIDS_cuml_GIT_REPO}",
+    "RAPIDS_cuopt_GIT_REPO": "${localEnv:RAPIDS_cuopt_GIT_REPO}",
+    "RAPIDS_cuvs_GIT_REPO": "${localEnv:RAPIDS_cuvs_GIT_REPO}",
+    "RAPIDS_kvikio_GIT_REPO": "${localEnv:RAPIDS_kvikio_GIT_REPO}",
+    "RAPIDS_LIBS": "${localEnv:RAPIDS_LIBS}",
+    "RAPIDS_nvforest_GIT_REPO": "${localEnv:RAPIDS_nvforest_GIT_REPO}",
+    "RAPIDS_raft_GIT_REPO": "${localEnv:RAPIDS_raft_GIT_REPO}",
+    "RAPIDS_rmm_GIT_REPO": "${localEnv:RAPIDS_rmm_GIT_REPO}",
+    "RAPIDS_ucxx_GIT_REPO": "${localEnv:RAPIDS_ucxx_GIT_REPO}",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE": "${localEnv:SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE:true}",
+    "SCCACHE_DIST_MAX_RETRIES": "${localEnv:SCCACHE_DIST_MAX_RETRIES:4}",
+    "SCCACHE_DIST_REQUEST_TIMEOUT": "${localEnv:SCCACHE_DIST_REQUEST_TIMEOUT:7140}",
+    "SCCACHE_IDLE_TIMEOUT": "${localEnv:SCCACHE_IDLE_TIMEOUT:0}",
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE": "true",
+    "SCCACHE_SERVER_LOG": "${localEnv:SCCACHE_SERVER_LOG:sccache=debug}"
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config,local/state} ${localWorkspaceFolder}/ci/rapids/.{conda,log/devcontainer-utils} ${localWorkspaceFolder}/ci/rapids/.repos/{rmm,kvikio,ucxx,cudf,rapidsmpf,raft,cuvs,nvforest,cuml,cugraph,cugraph-gnn,cuopt}"
+  ],
+  "postCreateCommand": [
+    "/bin/bash",
+    "-c",
+    "if [ ${CI:-false} = 'false' ]; then . /home/coder/cccl/ci/rapids/post-create-command.sh; fi; if test -z \"${DISABLE_SCCACHE:+x}\"; then echo \"export SCCACHE_DIST_URL='https://$(uname -m | sed -e 's/x86_/amd/' -e 's/aarch/arm/').linux.sccache.rapids.nvidia.com'\" >> /home/coder/.bashrc; fi"
+  ],
+  "postAttachCommand": [
+    "/bin/bash",
+    "-c",
+    "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; fi"
+  ],
+  "workspaceFolder": "/home/coder/cccl",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cccl,type=bind",
+  "mounts": [
+    "source=/etc/timezone,target=/etc/timezone,type=bind",
+    "source=/etc/localtime,target=/etc/localtime,type=bind",
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind",
+    "source=${localWorkspaceFolder}/ci/rapids/.repos/rmm,target=/home/coder/rmm,type=bind",
+    "source=${localWorkspaceFolder}/ci/rapids/.repos/kvikio,target=/home/coder/kvikio,type=bind",
+    "source=${localWorkspaceFolder}/ci/rapids/.repos/ucxx,target=/home/coder/ucxx,type=bind",
+    "source=${localWorkspaceFolder}/ci/rapids/.repos/cudf,target=/home/coder/cudf,type=bind",
+    "source=${localWorkspaceFolder}/ci/rapids/.repos/rapidsmpf,target=/home/coder/rapidsmpf,type=bind",
+    "source=${localWorkspaceFolder}/ci/rapids/.repos/raft,target=/home/coder/raft,type=bind",
+    "source=${localWorkspaceFolder}/ci/rapids/.repos/cuvs,target=/home/coder/cuvs,type=bind",
+    "source=${localWorkspaceFolder}/ci/rapids/.repos/nvforest,target=/home/coder/nvforest,type=bind",
+    "source=${localWorkspaceFolder}/ci/rapids/.repos/cuml,target=/home/coder/cuml,type=bind",
+    "source=${localWorkspaceFolder}/ci/rapids/.repos/cugraph,target=/home/coder/cugraph,type=bind",
+    "source=${localWorkspaceFolder}/ci/rapids/.repos/cugraph-gnn,target=/home/coder/cugraph-gnn,type=bind",
+    "source=${localWorkspaceFolder}/ci/rapids/.repos/cuopt,target=/home/coder/cuopt,type=bind",
+    "source=${localWorkspaceFolder}/ci/rapids/.conda,target=/home/coder/.conda,type=bind",
+    "source=${localWorkspaceFolder}/ci/rapids/.log/devcontainer-utils,target=/var/log/devcontainer-utils,type=bind"
+  ],
+  "features": {
+    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:26.10": {}
+  },
+  "overrideFeatureInstallOrder": [
+    "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils"
+  ],
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "augustocdias.tasks-shell-input",
+        "ms-python.flake8",
+        "nvidia.nsight-vscode-edition"
+      ],
+      "files.watcherExclude": {
+        "**/build/**": true,
+        "**/_skbuild/**": true,
+        "**/target/**": true,
+        "/home/coder/.aws/**/*": true,
+        "/home/coder/.cache/**/*": true,
+        "/home/coder/.conda/**/*": true,
+        "/home/coder/.local/share/**/*": true,
+        "/home/coder/.vscode-server/**/*": true
+      },
+      "search.exclude": {
+        "**/build/**": true,
+        "**/_skbuild/**": true,
+        "**/*.code-search": true,
+        "/home/coder/.aws/**/*": true,
+        "/home/coder/.cache/**/*": true,
+        "/home/coder/.conda/**/*": true,
+        "/home/coder/.local/share/**/*": true,
+        "/home/coder/.vscode-server/**/*": true
+      }
+    }
+  }
+}
--- a/cccl_upstream/ci/rapids/post-create-command.sh
+++ b/cccl_upstream/ci/rapids/post-create-command.sh
@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+
+PROJECT_MANIFEST_YML="${PROJECT_MANIFEST_YML:-"/opt/rapids-build-utils/manifest.yaml"}"
+
+_restore_original_manifest() {
+    if ! test -f "/tmp/manifest.yaml.orig"; then
+        cp "${PROJECT_MANIFEST_YML}" /tmp/manifest.yaml.orig;
+    fi
+    sudo cp /tmp/manifest.yaml.orig "${PROJECT_MANIFEST_YML}";
+}
+
+_apply_manifest_modifications() {
+    # Restore the original manifest.yaml
+    _restore_original_manifest;
+    # Remove unnecessary libs from manifest.yaml
+    _prune_libs_from_manifest;
+    # Update manifest.yaml repo git info
+    _update_repo_git_info;
+    # Create rapids-cmake override JSON file and update default CMake arguments in manifest.yaml
+    _create_rapids_cmake_override_json;
+    # Print the entire manifest.yaml after modifications
+    cat "${PROJECT_MANIFEST_YML}";
+    # Regenerate the RAPIDS build scripts from the new manifest.yaml
+    rapids-generate-scripts;
+}
+
+_prune_libs_from_manifest() {
+    local -;
+    set -euo pipefail;
+    if test -n "${RAPIDS_LIBS-}"; then
+        local -a filters="(${RAPIDS_LIBS})";
+        # prefix each element
+        filters=("${filters[@]/#/'"'}");
+        # suffix each element
+        filters=("${filters[@]/%/'",'}");
+        # Remove trailing comma
+        local -r filters_str="$(cut -d',' -f1-${#filters[@]} <<< "${filters[*]}")";
+        sudo yq -i ".repos |= filter(.cpp[].name | contains(${filters_str}))" "${PROJECT_MANIFEST_YML}";
+    fi
+}
+
+# shellcheck disable=SC2016
+_update_repo_git_info() {
+    local -;
+    set -euo pipefail;
+    yq '.repos[].name' "${PROJECT_MANIFEST_YML}" \
+  | xargs -r -n1 bash -c '
+    var="RAPIDS_${1//-/_}_GIT_REPO";
+    if test -v "${var}" && test -n "${!var}"; then
+        sudo yq -i "(.repos[] | select(.name == \"$1\") | .git) *= ${!var}" "${0}";
+    fi' "${PROJECT_MANIFEST_YML}";
+}
+
+_create_rapids_cmake_override_json() {
+    local -;
+    set -euo pipefail;
+    local rapids_cmake_tag=;
+    local rapids_cmake_upstream=;
+    if test -n "${RAPIDS_cmake_GIT_REPO-}"; then
+        rapids_cmake_tag="$(jq -r '.tag' <<< "${RAPIDS_cmake_GIT_REPO}")";
+        rapids_cmake_upstream="$(jq -r '.upstream' <<< "${RAPIDS_cmake_GIT_REPO}")";
+    else
+        rapids_cmake_tag="$(yq '.x-git-defaults.tag' /opt/rapids-build-utils/manifest.yaml)";
+        rapids_cmake_upstream="$(yq '.x-git-defaults.upstream' /opt/rapids-build-utils/manifest.yaml)";
+    fi
+
+    # Define CCCL_TAG to override the default CCCL SHA. Otherwise the current HEAD of the local checkout is used.
+    if test -n "${CCCL_TAG-}"; then
+        # If CCCL_TAG is defined, fetch it to the local checkout
+        git -C "${HOME}/cccl" fetch origin "${CCCL_TAG}";
+        cccl_sha="$(git -C "${HOME}/cccl" rev-parse FETCH_HEAD)";
+    else
+        cccl_sha="$(git -C "${HOME}/cccl" rev-parse HEAD)";
+    fi
+    echo "CCCL_VERSION: ${CCCL_VERSION-}";
+    echo "CCCL_TAG: ${CCCL_TAG-}";
+    echo "cccl_sha: ${cccl_sha}";
+
+    echo
+    echo "Replacing CCCL repo information in rapids-cmake versions.json:";
+    curl -fsSL -o- "https://raw.githubusercontent.com/${rapids_cmake_upstream}/rapids-cmake/${rapids_cmake_tag}/rapids-cmake/cpm/versions.json" \
+  | jq -r ".packages.CCCL *= {\"git_url\": \"${HOME}/cccl\", \"git_tag\": \"${cccl_sha}\", \"always_download\": true}" \
+  | jq -r "del(.packages.CCCL.url) | del(.packages.CCCL.url_hash)" \
+  > ~/rapids-cmake-override-versions-cccl-repo.json;
+
+    if test -n "${CCCL_VERSION-}"; then
+        echo "Patching CCCL_VERSION in rapids-cmake versions.json:";
+        jq -r ".packages.CCCL.version = \"${CCCL_VERSION}\"" ~/rapids-cmake-override-versions-cccl-repo.json \
+      > ~/rapids-cmake-override-versions.json;
+    else
+        echo "Using the default CCCL version in rapids-cmake versions.json:";
+        mv ~/rapids-cmake-override-versions-cccl-repo.json ~/rapids-cmake-override-versions.json;
+    fi
+
+    echo
+    echo "Final rapids-cmake-override-versions.json:";
+    cat ~/rapids-cmake-override-versions.json;
+
+    # Define default CMake args for each repo
+    local -a cmake_args=(BUILD_TESTS BUILD_BENCHMARKS BUILD_PRIMS_BENCH BUILD_CUGRAPH_MG_TESTS);
+    # Enable tests
+    cmake_args=("${cmake_args[@]/#/"-D"}");
+    cmake_args=("${cmake_args[@]/%/"=${RAPIDS_ENABLE_TESTS:-ON}"}");
+
+    # Always build RAFT shared lib
+    cmake_args+=("-DBUILD_SHARED_LIBS=ON");
+    cmake_args+=("-DRAFT_COMPILE_LIBRARY=ON");
+
+    # Tell rapids-cmake to use custom CCCL and cuCollections forks
+    cmake_args+=("-Drapids-cmake-branch=${rapids_cmake_tag}");
+    cmake_args+=("-Drapids-cmake-repo=${rapids_cmake_upstream}/rapids-cmake");
+    cmake_args+=("-DRAPIDS_CMAKE_CPM_DEFAULT_VERSION_FILE=${HOME}/rapids-cmake-override-versions.json");
+
+    # Inject the CMake args into manifest.yaml
+    sudo yq -i "(.repos[] | .cpp[] | .args.cmake) += \" ${cmake_args[*]}\"" "${PROJECT_MANIFEST_YML}";
+    sudo yq -i "(.repos[] | .python[] | .args.cmake) += \" ${cmake_args[*]}\"" "${PROJECT_MANIFEST_YML}";
+}
+
+_run_post_create_command() {
+    local -;
+    set -e;
+
+    # Install `rapids-build-utils` feature if it's not installed (i.e. if running locally via `.devcontainer/launch.sh -d`)
+    if ! test -f "${PROJECT_MANIFEST_YML}"; then
+        git clone --depth 1 --filter=blob:none --sparse https://github.com/rapidsai/devcontainers.git /tmp/rapidsai-devcontainers;
+        git -C /tmp/rapidsai-devcontainers sparse-checkout set features/src/rapids-build-utils;
+        (
+            cd /tmp/rapidsai-devcontainers/features/src/rapids-build-utils;
+            sudo bash ./install.sh;
+        )
+        rm -rf /tmp/rapidsai-devcontainers;
+    fi
+
+    # Modify manifest.yaml based on envvars
+    _apply_manifest_modifications;
+
+    # Clone all the repos
+    gh config set git_protocol https;
+    gh config set git_protocol https --host github.com;
+
+    clone-all -j "$(nproc --all)" -v -q --clone-upstream --single-branch --shallow-submodules --no-update-env;
+}
+
+if [[ "$(basename "${BASH_SOURCE[${#BASH_SOURCE[@]}-1]}")" = post-create-command.sh ]]; then
+    _run_post_create_command;
+fi
--- a/cccl_upstream/ci/rapids/rapids-entrypoint.sh
+++ b/cccl_upstream/ci/rapids/rapids-entrypoint.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC1091
+
+set -e;
+
+if [[ -n "${GITHUB_ACTIONS:-}" ]]; then
+    echo "::group::Cloning RAPIDS..."
+fi
+
+ci/rapids/post-create-command.sh;
+rapids-post-start-command -f;
+
+if [[ -n "${GITHUB_ACTIONS:-}" ]]; then
+    echo "::endgroup::"
+fi
+
+if test $# -gt 0; then
+    exec "$@";
+else
+    exec /bin/bash -li;
+fi