dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat(cccl): integrate missing CCCL directories — python/, ci/, .agent/, docs/, test/

Browse Source 
Sparse-checkout from NVIDIA/cccl main branch to complete cccl_upstream:

Added:
- python/cuda_cccl/ (226 files) — Python bindings for device-level algorithms
  Critical for muh toolchain: cuda.compute.reduce_into, scan, radix_sort, etc.
  Includes 204 .py files with full test coverage for all 27 algorithms
- ci/ (163 files) — Build/test infrastructure
  build_cub.sh, test_cub.sh, build_and_test_targets.sh, matrix.yaml
  Directly maps to our [INFRA-CI] and [INFRA-BUILD] items
- .agent/skills/ (7 files) — NVIDIA's own agent skills for CCCL
  cccl-style/SKILL.md, cccl-test/SKILL.md, sass-diff/SKILL.md
- docs/ (491 files) — Official CCCL documentation
  CI references, CMake guides, Python compute docs, libcudacxx PTX docs
- test/ (12 files) — Top-level integration tests (cuda_smoke, stdpar)
- Root configs: .clang-format, .clang-tidy, CONTRIBUTING.md, pyproject.toml
- CLAUDE.md symlink → AGENTS.md (NVIDIA's standard)

cccl_upstream now mirrors full NVIDIA/cccl structure:
  Before: 42M (cub + thrust + libcudacxx + cudax + c + examples + benchmarks)
  After:  53M (+python +ci +docs +.agent +test +configs)

This completes the CCCL base needed for:
- [muh-bench] items: ci/util/build_and_test_targets.sh for targeted builds
- [CCCL-verify] items: python/cuda_cccl/tests/ as reference implementations
- [CCCL-test] items: ci/test_cub.sh, ci/test_thrust.sh
- Agent workflow: .agent/skills/ for consistent style and test patterns
This commit is contained in:
muh-bot
2026-08-07 02:34:33 +00:00
parent 3f97dca7ad
commit 2a7ca101d7
908 changed files with 121615 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
cccl_upstream/ci/util/artifacts/common.sh Executable file
View File

@@ -0,0 +1,39 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
echo "This script must be sourced, not executed directly." >&2
exit 1
fi
if [[ -z "${GITHUB_ACTIONS:-}" ]]; then
echo "This script must be run in a GitHub Actions environment." >&2
exit 1
fi
to_posix_path() {
local path="$1"
if [[ "$path" =~ ^([A-Za-z]):([\\/]?.*)$ ]]; then
local drive="${BASH_REMATCH[1]}"
local rest="${BASH_REMATCH[2]}"
rest="${rest//\\/\/}"
printf '/%s%s\n' "${drive,,}" "$rest"
return
fi
printf '%s\n' "$path"
}
runner_temp_posix="$(to_posix_path "${RUNNER_TEMP:-/tmp}")"
export ARTIFACT_UPLOAD_STAGE="${runner_temp_posix}/artifact_upload_stage"
export ARTIFACT_ARCHIVES="${runner_temp_posix}/artifact_archives"
export ARTIFACT_UPLOAD_REGISTERY="${ARTIFACT_UPLOAD_STAGE}/artifact_upload_registry.json"
mkdir -p "$ARTIFACT_UPLOAD_STAGE" "$ARTIFACT_ARCHIVES"
if [[ ! -f "$ARTIFACT_UPLOAD_REGISTERY" ]]; then
echo "[]" > "$ARTIFACT_UPLOAD_REGISTERY"
fi

40
cccl_upstream/ci/util/artifacts/download.sh Executable file
View File

@@ -0,0 +1,40 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Usage: $0 <name> [<path>]
Download artifacts uploaded by other jobs in this CI run.
Example Usage:
Download an artifact to the current directory:
$0 source_artifact.tar.gz
Download a packed artifact and extract it to the provided path:
$0 job-\$ID-products some/path/
EOF
)
readonly usage
if [[ "$#" -lt 1 ]]; then
echo "Error: Missing artifact name." >&2
echo "$usage" >&2
exit 1
fi
readonly artifact_name="$1"
if [[ "$#" -eq 1 ]]; then
artifact_path="./"
else
artifact_path="$2"
fi
start=$SECONDS
"$ci_dir/util/artifacts/download/fetch.sh" "$artifact_name" "$artifact_path"
echo "Artifact '$artifact_name' downloaded to '$artifact_path' in $((SECONDS - start)) seconds."

40
cccl_upstream/ci/util/artifacts/download/fetch.sh Executable file
View File

@@ -0,0 +1,40 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Usage: $0 <artifact_name> <target_directory>
Downloads files from a named artifact from the current CI workflow run into the specified directory.
Example Usages:
- $0 my_artifact.tar.gz ./
- $0 my_artifact /path/to/some/directory/
EOF
)
readonly usage
if [[ "$#" -lt 2 ]]; then
echo "Error: Missing arguments." >&2
echo "$usage" >&2
exit 1
fi
readonly artifact_name="$1"
# Create the target directory and then get its absolute path
mkdir -p "$2"
target_directory="$(cd "$2" && pwd)"
readonly target_directory
echo "Downloading artifact '$artifact_name' to '$target_directory'"
# shellcheck disable=SC2154
"$ci_dir/util/retry.sh" 5 30 \
gh run download "${GITHUB_RUN_ID}" \
--name "$artifact_name" \
--dir "$target_directory"

45
cccl_upstream/ci/util/artifacts/download/unpack.sh Executable file
View File

@@ -0,0 +1,45 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Usage: $0 <artifact_name> <artifact_path>
Unpacks a fetched packed artifact's tar.zst archive into the specified directory.
Example Usages:
- $0 /tmp/my_artifact.tar.zst /tmp/my_artifact
- $0 /path/to/archive.tar.zst /path/to/extract/
EOF
)
readonly usage
if [[ "$#" -lt 2 ]]; then
echo "Error: Missing arguments." >&2
echo "$usage" >&2
exit 1
fi
if ! command -v zstd > /dev/null 2>&1; then
echo "Error: zstd not found." >&2
exit 1
fi
readonly artifact_name="$1"
readonly artifact_path="$2"
readonly artifact_archive="$ARTIFACT_ARCHIVES/${artifact_name}.tar.zst"
echo "Unpacking artifact from '$artifact_archive' to '$artifact_path'"
echo "Using zstd executable: $(command -v zstd)"
# Create the artifact path directory if it doesn't exist
mkdir -p "$artifact_path"
zstd --decompress --threads=0 --stdout "$artifact_archive" \
| tar -xv -C "$artifact_path"

45
cccl_upstream/ci/util/artifacts/download_packed.sh Executable file
View File

@@ -0,0 +1,45 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Usage: $0 <name> [<path>]
Download and extracts a packed artifact uploaded by another job in this CI run.
Example Usage:
Download an artifact to the current directory:
$0 source_artifact.tar.gz
Download a packed artifact and extract it to the provided path:
$0 job-\$ID-products build/
EOF
)
readonly usage
if [[ "$#" -lt 1 ]]; then
echo "Error: Missing artifact name." >&2
echo "$usage" >&2
exit 1
fi
readonly artifact_name="$1"
if [[ "$#" -eq 1 ]]; then
artifact_path="./"
else
artifact_path="$2"
fi
start=$SECONDS
"$ci_dir/util/artifacts/download/fetch.sh" "$artifact_name" "${ARTIFACT_ARCHIVES}"
fetched=$SECONDS
"$ci_dir/util/artifacts/download/unpack.sh" "$artifact_name" "$artifact_path"
unpacked=$SECONDS
echo "Artifact '$artifact_name' fetched in $((fetched - start)) seconds."
echo "Artifact '$artifact_name' unpacked in $((unpacked - fetched)) seconds."

70
cccl_upstream/ci/util/artifacts/stage.sh Executable file
View File

@@ -0,0 +1,70 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Usage: $0 <artifact_name> <regex> [<regex> ...]
Stages files matching the provided regexes path for upload under the specified artifact.
Regexes are passed to the 'find' command's -regex option within the artifact stage path and implicitly
start with '^\\./'.
Staged files can be unstaged using the 'artifact/unstage.sh' script.
All stage / unstage operations on the same artifact must be performed from the same working directory.
Once a stage is complete, 'artifact/upload_stage_packed.sh' can be used to create a packed artifact
from the stage. See also 'artifact/upload/pack.sh' and 'artifact/upload/build.sh' for more staging options.
Example Usage:
Stage built binaries and .cmake files in \${ARTIFACT_UPLOAD_STAGE}/test_artifacts for upload:
$0 test_artifacts 'bin/.*' 'lib/.*' '.*cmake$'
EOF
)
readonly usage
if [[ "$#" -lt 2 ]]; then
echo "Error: Missing arguments." >&2
echo "$usage" >&2
exit 1
fi
artifact_name="$1"
shift
regexes=("$@")
artifact_stage_path="${ARTIFACT_UPLOAD_STAGE}/${artifact_name}"
if [[ "$artifact_stage_path" != /* ]]; then
artifact_stage_path="$(pwd)/$artifact_stage_path"
fi
mkdir -p "$artifact_stage_path"
artifact_index_file="$artifact_stage_path/artifact_index.txt"
artifact_index_cwd="$artifact_stage_path/artifact_index_cwd.txt"
if [[ -f "$artifact_index_cwd" ]]; then
# Check that the cwd matches the original staging directory if the index already exists:
if [[ "$(cat "$artifact_index_cwd")" != "$(pwd)" ]]; then
echo "Error: The current working directory has changed since the artifact was staged." >&2
echo "Cannot currently stage files from multiple source directories." >&2
exit 1
fi
else
pwd > "$artifact_index_cwd"
fi
echo "Staging artifacts in '$artifact_stage_path'"
for regex in "${regexes[@]}"; do
# Prepend './' to the regex for convenience. There's an implied ^ at the start of the find regex,
# and paths always start with ./, so this lets us match top level files directly.
regex="\\./$regex"
echo "Staging files matching regex: $regex"
find . -type f -regex "$regex" | tee -a "$artifact_index_file"
echo
done

56
cccl_upstream/ci/util/artifacts/unstage.sh Executable file
View File

@@ -0,0 +1,56 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Usage: $0 <artifact_name> <regex> [<regex> ...]
Unstages (removes) files matching the provided regexes from the specified artifact stage.
Regexes follow the same rules as artifact/stage.sh.
This may be used to remove files that were previously staged for upload before packing or building the artifacts.
Example Usage:
Unstage previously-staged built binaries and .cmake files from test_artifacts:
$0 test_artifacts 'bin/.*' 'lib/.*' '.*cmake$'
EOF
)
readonly usage
if [[ "$#" -lt 2 ]]; then
echo "Error: Missing arguments." >&2
echo "$usage" >&2
exit 1
fi
artifact_name="$1"
shift
regexes=("$@")
artifact_stage_path="${ARTIFACT_UPLOAD_STAGE}/${artifact_name}"
if [[ "$artifact_stage_path" != /* ]]; then
artifact_stage_path="$(pwd)/$artifact_stage_path"
fi
mkdir -p "$artifact_stage_path"
artifact_index_file="$artifact_stage_path/artifact_index.txt"
artifact_index_cwd="$artifact_stage_path/artifact_index_cwd.txt"
pwd > "$artifact_index_cwd"
echo "Unstaging artifacts in '$artifact_stage_path'"
for regex in "${regexes[@]}"; do
# Modify regex for consistency with staging script:
regex="^\\./$regex"
echo "Unstaging files matching regex: $regex"
grep -E "$regex" "$artifact_index_file"
grep -v -E "$regex" "$artifact_index_file" > "${artifact_index_file}.tmp" && \
mv "${artifact_index_file}.tmp" "$artifact_index_file"
done

59
cccl_upstream/ci/util/artifacts/upload.sh Executable file
View File

@@ -0,0 +1,59 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Usage: $0 <name> [<regex> ...]
Creates an artifact consisting of a zip file containing a single file or set of regex matches.
Regexes are passed to the $(command -v find) command's -regex option in the current directory.
'./' is prepended to all regexes for convenience.
The artifact will contain all matching files with paths relative to the current directory.
If no regexes are provided, the artifact will be created from a file in the current directory.
The file must have the same name as the artifact.
Example Usage:
Create an artifact of the given file in the current directory using the filename as the artifact name:
$0 some_resource.log
Copy all files that match the regexes to a staging directory, and upload a artifact that zips this directory.
$0 job-\$JOB_ID-products \
'build\.ninja$' \
'.*rules\.ninja$' \
'CMakeCache\.txt$' \
'.*VerifyGlobs\.cmake$' \
'.*CTestTestfile\.cmake$' \
'bin/.*' \
'lib/.*'
EOF
)
readonly usage
if [[ "$#" -lt 1 ]]; then
echo "Error: Missing artifact name." >&2
echo "$usage" >&2
exit 1
fi
readonly artifact_name="$1"
# If no regexes are provided, use the artifact name as the path:
if [[ "$#" -eq 1 ]]; then
"$ci_dir/util/artifacts/upload/register.sh" "$artifact_name" "$artifact_name"
exit
fi
shift
"$ci_dir/util/artifacts/stage.sh" "$artifact_name" "$@" > /dev/null
"$ci_dir/util/artifacts/upload_stage.sh" "$artifact_name"

44
cccl_upstream/ci/util/artifacts/upload/build.sh Executable file
View File

@@ -0,0 +1,44 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Usage: $0 <artifact_name>
Builds a physical tree containing a staged artifact created using artifact/stage.sh / unstage.sh.
The artifact root will be located at \${ARTIFACT_UPLOAD_STAGE}/<artifact_name>/<artifact_name>.
EOF
)
readonly usage
if [[ "$#" -ne 1 ]]; then
echo "Error: Invalid number of arguments." >&2
echo "$usage" >&2
exit 1
fi
readonly artifact_name="$1"
readonly artifact_stage_path="${ARTIFACT_UPLOAD_STAGE}/${artifact_name}"
readonly artifact_index_file="$artifact_stage_path/artifact_index.txt"
readonly artifact_cwd_file="$artifact_stage_path/artifact_index_cwd.txt"
readonly artifact_dir="${ARTIFACT_UPLOAD_STAGE}/${artifact_name}/${artifact_name}"
artifact_cwd="$(cat "$artifact_cwd_file")"
readonly artifact_cwd
mkdir -p "$artifact_dir"
echo "Building artifact '$artifact_name' in '$artifact_dir'"
echo "Pulling artifacts from working directory: $artifact_cwd"
(
cd "$artifact_cwd"
while IFS= read -r file; do
cp -v --parents "$file" "$artifact_dir"
done < "$artifact_index_file"
)

47
cccl_upstream/ci/util/artifacts/upload/pack.sh Executable file
View File

@@ -0,0 +1,47 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Usage: $0 <artifact_name>
Packs a staged artifact (created using artifact/stage.sh) into a tar.zst archive.
The archive will be generated from the staged index file in \${ARTIFACT_UPLOAD_STAGE}/<artifact_name> and
saved to \${ARTIFACT_UPLOAD_STAGE}/<artifact_name>/<artifact_name>.tar.zst.
Example Usages:
- $0 test_artifact
EOF
)
readonly usage
if [[ "$#" -ne 1 ]]; then
echo "Error: Invalid number of arguments." >&2
echo "$usage" >&2
exit 1
fi
if ! command -v zstd > /dev/null 2>&1; then
echo "Error: zstd not found." >&2
exit 1
fi
readonly artifact_name="$1"
readonly artifact_stage_path="${ARTIFACT_UPLOAD_STAGE}/${artifact_name}"
readonly artifact_index_file="$artifact_stage_path/artifact_index.txt"
readonly artifact_cwd_file="$artifact_stage_path/artifact_index_cwd.txt"
readonly artifact_archive="${ARTIFACT_UPLOAD_STAGE}/${artifact_name}/${artifact_name}.tar.zst"
echo "Packing artifact '$artifact_stage_path' into '$artifact_archive'"
echo "Using zstd: $(command -v zstd)"
echo "Pulling artifacts from working directory: $(cat "$artifact_cwd_file")"
tar -cv -C "$(cat "$artifact_cwd_file")" -T "$artifact_index_file" \
| zstd --compress --threads=0 \
> "$artifact_archive"

21
cccl_upstream/ci/util/artifacts/upload/print_matrix.sh Executable file
View File

@@ -0,0 +1,21 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Prints the Github Actions matrix for uploading all registered artifacts.
EOF
)
readonly usage
if [[ "$#" -ne 0 ]]; then
echo "$usage" >&2
exit 1
fi
jq -c '.' "${ARTIFACT_UPLOAD_REGISTERY:?}"

61
cccl_upstream/ci/util/artifacts/upload/register.sh Executable file
View File

@@ -0,0 +1,61 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
readonly artifact_compression_level=6
readonly artifact_retention_days=7
usage=$(cat <<EOF
Usage: $0 <artifact_name> [<artifact_path>]
Registers artifacts for upload. If path is not provided, it defaults to the artifact name.
Compression level is set to $artifact_compression_level by default.
Use 'upload/set_compression_level.sh' to change this after registering if needed.
Default retention days is set to $artifact_retention_days.
Use 'upload/set_retention_days.sh' after registering to change this if needed.
Example Usages:
- $0 my_artifact.tar.gz # Assumes the artifact is in the current directory.
- $0 my_artifact /path/to/my_artifact.tar.gz
- $0 my_artifact /path/to/my_artifact_directory/
EOF
)
readonly usage
if [[ "$#" -lt 1 ]]; then
echo "Error: Missing artifact name." >&2
echo "$usage" >&2
exit 1
fi
artifact_name="$1"
artifact_path="${2:-$artifact_name}"
# Ensure the artifact path is absolute
if [[ "$artifact_path" != /* ]]; then
artifact_path="$(pwd)/$artifact_path"
fi
if [[ ! -e "$artifact_path" ]]; then
echo "Error: Artifact path '$artifact_path' does not exist." >&2
echo "$usage" >&2
exit 1
fi
# Register the artifact:
jq --arg name "$artifact_name" \
--arg path "$artifact_path" \
--arg retention_days "$artifact_retention_days" \
--argjson compression_level "$artifact_compression_level" \
'. += [{"name": $name, "path": $path, "retention_days": ($retention_days | tonumber), "compression_level": $compression_level}]' \
"$ARTIFACT_UPLOAD_REGISTERY" > "$ARTIFACT_UPLOAD_REGISTERY.tmp" && \
mv "$ARTIFACT_UPLOAD_REGISTERY.tmp" "$ARTIFACT_UPLOAD_REGISTERY"
echo "Artifact '$artifact_name' registered for upload with path '$artifact_path'."

35
cccl_upstream/ci/util/artifacts/upload/set_compression_level.sh Executable file
View File

@@ -0,0 +1,35 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Usage: $0 <artifact_name> <compression_level>
Sets the compression level for an artifact registered for upload.
Example Usage:
$0 some_huge_precompressed_archive 0
$0 some_many_small_uncompressed_files 10
EOF
)
readonly usage
if [[ "$#" -lt 2 ]]; then
echo "Error: Missing arguments." >&2
echo "$usage" >&2
exit 1
fi
artifact_name="$1"
compression_level="$2"
# Find the artifact entry and update its compression level
jq --arg name "$artifact_name" --argjson compression_level "$compression_level" \
'map(if .name == $name then .compression_level = $compression_level else . end)' \
"$ARTIFACT_UPLOAD_REGISTERY" > "$ARTIFACT_UPLOAD_REGISTERY.tmp" && \
mv "$ARTIFACT_UPLOAD_REGISTERY.tmp" "$ARTIFACT_UPLOAD_REGISTERY"

36
cccl_upstream/ci/util/artifacts/upload/set_retention_days.sh Executable file
View File

@@ -0,0 +1,36 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Usage: $0 <artifact_name> <retention_days>
Sets the retention days for an artifact registered for upload.
Example Usage:
$0 some_huge_temporary_artifact 1
$0 some_small_useful_output 7
$0 some_long_term_artifact 30
EOF
)
readonly usage
if [[ "$#" -lt 2 ]]; then
echo "Error: Missing arguments." >&2
echo "$usage" >&2
exit 1
fi
artifact_name="$1"
retention_days="$2"
# Find the artifact entry and update its retention days
jq --arg name "$artifact_name" --argjson retention_days "$retention_days" \
'map(if .name == $name then .retention_days = $retention_days else . end)' \
"$ARTIFACT_UPLOAD_REGISTERY" > "$ARTIFACT_UPLOAD_REGISTERY.tmp" && \
mv "$ARTIFACT_UPLOAD_REGISTERY.tmp" "$ARTIFACT_UPLOAD_REGISTERY"

54
cccl_upstream/ci/util/artifacts/upload_packed.sh Executable file
View File

@@ -0,0 +1,54 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Usage: $0 <name> <regex> [<regex> ...]
Create a compressed artifact, suitable for large, temporary files such as build products or test binaries
that need to be quickly uploaded and downloaded between CI jobs. The artifact will exist of a
zip file containing an <artifact_name>.tar.zst archive, packed with the parallel zstd.
Regexes are passed to the $(command -v find) command's -regex option in the current directory.
'./' is prepended to all regexes for convenience.
The artifact will contain all matching files relative to the current directory.
If no regexes are provided, the artifact will be created from a file in the current directory.
The file must have the same name as the artifact.
Example Usage:
Create an artifact of the given file in the current directory using the filename as the artifact name:
$0 some_resource.log
Copy all files that match the regexes to a staging directory, and upload a artifact that zips this directory.
$0 job-\$JOB_ID-products \
'build\.ninja$' \
'.*rules\.ninja$' \
'CMakeCache\.txt$' \
'.*VerifyGlobs\.cmake$' \
'.*CTestTestfile\.cmake$' \
'bin/.*' \
'lib/.*'
EOF
)
readonly usage
if [[ "$#" -lt 2 ]]; then
echo "Error: Invalid number of arguments." >&2
echo "$usage" >&2
exit 1
fi
readonly artifact_name="$1"
shift
"$ci_dir/util/artifacts/stage.sh" "$artifact_name" "$@" > /dev/null
"$ci_dir/util/artifacts/upload_stage_packed.sh" "$artifact_name"

31
cccl_upstream/ci/util/artifacts/upload_stage.sh Executable file
View File

@@ -0,0 +1,31 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Usage: $0 <name>
Same as 'ci/util/artifacts/upload_packed.sh', but assumes that the stage has already been created using
'ci/util/artifacts/stage.sh' and 'unstage.sh'. Performs the packing and registration steps only.
EOF
)
readonly usage
if [[ "$#" -ne 1 ]]; then
echo "Error: Invalid number of arguments." >&2
echo "$usage" >&2
exit 1
fi
readonly artifact_name="$1"
readonly artifact_dir="$ARTIFACT_UPLOAD_STAGE/${artifact_name}/${artifact_name}"
start=$SECONDS
"$ci_dir/util/artifacts/upload/build.sh" "$artifact_name"
"$ci_dir/util/artifacts/upload/register.sh" "$artifact_name" "$artifact_dir"
echo "Artifact '$artifact_name' built in $((SECONDS - start)) seconds."

34
cccl_upstream/ci/util/artifacts/upload_stage_packed.sh Executable file
View File

@@ -0,0 +1,34 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
usage=$(cat <<EOF
Usage: $0 <name>
Same as 'ci/util/artifacts/upload_packed.sh', but assumes that the stage has already been created using
'ci/util/artifacts/stage.sh' and 'unstage.sh'. Performs the packing and registration steps only.
EOF
)
readonly usage
if [[ "$#" -ne 1 ]]; then
echo "Error: Invalid number of arguments." >&2
echo "$usage" >&2
exit 1
fi
readonly artifact_name="$1"
# shellcheck disable=SC2154
readonly artifact_archive="$ARTIFACT_UPLOAD_STAGE/${artifact_name}/${artifact_name}.tar.zst"
start=$SECONDS
"$ci_dir/util/artifacts/upload/pack.sh" "$artifact_name"
"$ci_dir/util/artifacts/upload/register.sh" "$artifact_name" "$artifact_archive"
# Already compressed while packing:
"$ci_dir/util/artifacts/upload/set_compression_level.sh" "$artifact_name" 0 > /dev/null
echo "Artifact '$artifact_name' packed in $((SECONDS - start)) seconds."

165
cccl_upstream/ci/util/build_and_test_targets.sh Executable file
View File

@@ -0,0 +1,165 @@
#!/usr/bin/env bash
set -euo pipefail
# This must be run from the cccl repo root, but the script may be relocated by git_bisect.sh.
# Check that the current directory looks like the repo root:
if [[ ! -f "./cccl-version.json" ]]; then
echo "This script must be run from the cccl repo root."
exit 1
fi
usage() {
cat <<USAGE
Usage: $0 [--preset NAME | --configure-override CMD] [options]
Options:
-h, --help Show this help and exit
--preset NAME CMake preset
--cmake-options STR Extra options passed to CMake preset configure (optional)
--configure-override CMD Command to run for configuration instead of cmake preset
If set, --preset and --cmake-options will be ignored
--build-targets STR Space separated ninja build targets (optional)
If omitted, no targets will be built -- explicitly specify 'all' if needed.
--ctest-targets STR Space separated CTest -R regex patterns (optional)
If omitted, no tests will be run -- explicitly specify '.' to run all.
--lit-precompile-tests STR Space-separated libcudacxx lit test paths to precompile without execution (optional)
e.g. 'cuda/utility/basic_any.pass.cpp'
--lit-tests STR Space-separated libcudacxx lit test paths to execute (optional)
e.g. 'cuda/utility/basic_any.pass.cpp'
--custom-test-cmd CMD Custom command run after build and tests (optional)
USAGE
}
start_timestamp=${SECONDS}
function elapsed_time {
local duration=$(( SECONDS - start_timestamp ))
local minutes=$(( duration / 60 ))
local seconds=$(( duration % 60 ))
printf "%dm%02ds" "$minutes" "$seconds"
}
PRESET=""
BUILD_TARGETS=()
CTEST_TARGETS=()
LIT_PRECOMPILE_TESTS=()
LIT_TESTS=()
CMAKE_OPTIONS=()
CONFIGURE_OVERRIDE=""
CUSTOM_TEST_CMD=""
while [[ $# -gt 0 ]]; do
case "$1" in
-h|--help) usage; exit 0 ;;
--preset) PRESET="${2:-}"; shift 2 ;;
--build-targets) declare -a BUILD_TARGETS="(${2:-})"; shift 2 ;;
--ctest-targets) declare -a CTEST_TARGETS="(${2:-})"; shift 2 ;;
--lit-precompile-tests) declare -a LIT_PRECOMPILE_TESTS="(${2:-})"; shift 2 ;;
--lit-tests) declare -a LIT_TESTS="(${2:-})"; shift 2 ;;
--cmake-options) declare -a CMAKE_OPTIONS="(${2:-})"; shift 2 ;;
--configure-override) CONFIGURE_OVERRIDE="${2:-}"; shift 2 ;;
--custom-test-cmd) CUSTOM_TEST_CMD="${2:-}"; shift 2 ;;
*) echo "Unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
if [[ -z "${PRESET}" && -z "${CONFIGURE_OVERRIDE}" ]]; then
echo "::error:: --preset or --configure-override is required" >&2
usage
exit 2
fi
if [[ -n "${CONFIGURE_OVERRIDE}" ]]; then
if [[ -n "${PRESET}" ]]; then
echo "::warning:: --preset ignored due to --configure-override" >&2
fi
if [[ "${#CMAKE_OPTIONS[@]}" -gt 0 ]]; then
echo "::warning:: --cmake-options ignored due to --configure-override" >&2
fi
fi
echo "::group::⚙️ Testing $(git log --oneline | head -n1)"
# Configure and parse the build directory from CMake output
BUILD_DIR=""
cmlog_file="$(mktemp /tmp/cmake-config-XXXXXX.log)"
if [[ -n "${CONFIGURE_OVERRIDE}" ]]; then
if ! (set -x; eval "${CONFIGURE_OVERRIDE}") 2>&1 | tee "${cmlog_file}"; then
echo "::endgroup::"
echo -e "🔴📝 Configuration override failed ($(elapsed_time)):\n\t${CONFIGURE_OVERRIDE}"
exit 1
fi
else
if ! (set -x; cmake --preset "${PRESET}" "${CMAKE_OPTIONS[@]}") 2>&1 | tee "${cmlog_file}"; then
echo "::endgroup::"
echo "🔴📝 CMake configure failed for preset ${PRESET} ($(elapsed_time))"
exit 1
fi
fi
BUILD_DIR=$(awk -F': ' '/-- Build files have been written to:/ {print $2}' "${cmlog_file}" | tail -n1)
if [[ -z "${BUILD_DIR}" ]]; then
echo "::endgroup::"
echo "🔴‼️ Unable to determine build directory ($(elapsed_time))"
exit 1
fi
if [[ "${#BUILD_TARGETS[@]}" -gt 0 ]]; then
if ! (set -x; ninja -C "${BUILD_DIR}" "${BUILD_TARGETS[@]}"); then
echo "::endgroup::"
echo "🔴🛠️ Ninja build failed for targets ($(elapsed_time)): ${BUILD_TARGETS[*]@Q}"
exit 1
fi
fi
if [[ "${#CTEST_TARGETS[@]}" -gt 0 ]]; then
for t in "${CTEST_TARGETS[@]}"; do
if ! (set -x; ctest --test-dir "${BUILD_DIR}" -R "$t" -V --output-on-failure); then
echo "::endgroup::"
echo "🔴🔎 CTest failed for target $t ($(elapsed_time))"
exit 1
fi
done
fi
if [[ "${#LIT_PRECOMPILE_TESTS[@]}" -gt 0 || "${#LIT_TESTS[@]}" -gt 0 ]]; then
lit_site_cfg="${BUILD_DIR}/libcudacxx/test/libcudacxx/lit.site.cfg"
if [[ ! -f "${lit_site_cfg}" ]]; then
echo "::endgroup::"
echo "🔴🧪 LIT site config not found ($(elapsed_time)): ${lit_site_cfg}"
exit 1
fi
fi
if [[ "${#LIT_PRECOMPILE_TESTS[@]}" -gt 0 ]]; then
for t in "${LIT_PRECOMPILE_TESTS[@]}"; do
t_path="libcudacxx/test/libcudacxx/${t}"
if ! (set -x; LIBCUDACXX_SITE_CONFIG="${lit_site_cfg}" lit -v "-Dexecutor=NoopExecutor()" "${t_path}"); then
echo "::endgroup::"
echo "🔴🧪 LIT precompile failed ($(elapsed_time)): ${t}"
exit 1
fi
done
fi
if [[ "${#LIT_TESTS[@]}" -gt 0 ]]; then
for t in "${LIT_TESTS[@]}"; do
t_path="libcudacxx/test/libcudacxx/${t}"
if ! (set -x; LIBCUDACXX_SITE_CONFIG="${lit_site_cfg}" lit -v "${t_path}"); then
echo "::endgroup::"
echo "🔴🧪 LIT test failed ($(elapsed_time)): ${t}"
exit 1
fi
done
fi
if [[ -n "${CUSTOM_TEST_CMD}" ]]; then
if ! (set -x; eval "${CUSTOM_TEST_CMD}"); then
echo "::endgroup::"
echo "🔴🧪 Custom test command failed ($(elapsed_time)): ${CUSTOM_TEST_CMD}"
exit 1
fi
fi
echo "::endgroup::"
echo "🟢✅ Passed ($(elapsed_time))"
exit 0

78
cccl_upstream/ci/util/create_mock_job_env.sh Executable file
View File

@@ -0,0 +1,78 @@
#!/usr/bin/env bash
set -euo pipefail
usage=$(cat <<EOF
Usage: $0 <run id> <job_id>
Allows the scripts in ci/util/workflow and ci/util/artifacts to run as though they are running in a CI environment.
Create a mock job environment for testing purposes.
The run id can be found in the workflow run URL, and the job_id can be found at the start of the Run Command job step.
A new shell is spawned with the remote environment of a specific job from a specific workflow run.
Environment variables are configured to mimic the CI environment.
Caches and previously downloaded artifacts in /tmp are deleted to ensure a clean state.
!! Note that this does affect the caller's filesystem:
/tmp/workflow
/tmp/<artifact stages, archives, registry>
and similar caches will be deleted **from the caller's filesystem**.
This is usually fine, but be might overwrite files in-use by other mock environments.
EOF
)
readonly usage
if [[ "$#" -ne 2 ]]; then
echo "Error: Invalid number of arguments." >&2
echo "$usage" >&2
exit 1
fi
if [[ -n "${GITHUB_ACTIONS:-}" ]]; then
echo "$0: Detected another GITHUB_ACTIONS environment." >&2
echo "unset GITHUB_ACTIONS if this is intentional." >&2
exit 1
fi
if [[ -z "${DEVCONTAINER_NAME:-}" ]]; then
echo "This script must be run inside a devcontainer." >&2
exit 1
else
echo "Running in devcontainer: $DEVCONTAINER_NAME"
fi
ci_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/../" && pwd)
export GITHUB_ACTIONS=true
export GITHUB_RUN_ID="$1"
export JOB_ID="$2"
(
# shellcheck source=ci/util/workflow/common.sh
source "$ci_dir/util/workflow/common.sh"
# shellcheck source=ci/util/artifacts/common.sh
source "$ci_dir/util/artifacts/common.sh"
rm -rf "$WORKFLOW_DIR"
rm -rf "$ARTIFACT_ARCHIVES"
rm -rf "$ARTIFACT_UPLOAD_STAGE"
rm -rf "$ARTIFACT_UPLOAD_REGISTERY"
)
# Configure shell prompt:
export PS0=""
export PS1="<Mock Job: $GITHUB_RUN_ID $JOB_ID> [\u@\h \W]$ "
export PROMPT_COMMAND=""
echo "Starting new shell for emulating Job $JOB_ID in Run $GITHUB_RUN_ID".
echo ""
bash --norc --noprofile -i || :
echo
echo "Exiting mock job environment."

69
cccl_upstream/ci/util/extract_switches.sh Executable file
View File

@@ -0,0 +1,69 @@
#!/usr/bin/env bash
# Similar to getopt, but only extracts recognized switches and leaves all other arguments in place.
#
# Example Usage:
# new_args=$(extract_switches.sh -cpu-only -gpu-only -- "$@")
# declare -a new_args="(${new_args})"
# set -- "${new_args[@]}"
# while true; do
# case "$1" in
# -cpu-only) CPU_ONLY=true; shift;;
# -gpu-only) GPU_ONLY=true; shift;;
# --) shift; break;;
# *) echo "Unknown argument: $1"; exit 1;;
# esac
# done
#
# This leaves all unrecognized arguments in $@ for later parsing.
# Parse switches
switches=()
for arg in "$@"; do
case "$arg" in
--help | -h)
cat <<"EOF" | cut -c 5-
Usage: extract_switches.sh <switch> [<switch> ...] -- <argv>
Sorts any recognized switches in argv to the front and returns the result.
Unrecognized switches are left in place.
Example Usage:
new_args="$(extract_switches.sh -cpu-only -gpu-only -- "$@")"
declare -a new_args="(${new_args})"
set -- "${new_args[@]}"
while true; do
case "$1" in
-cpu-only) CPU_ONLY=true; shift;;
-gpu-only) GPU_ONLY=true; shift;;
--) shift; break;;
*) echo "Unknown argument: $1"; exit 1;;
esac
done
EOF
exit
;;
--)
shift
break
;;
*)
switches+=("$arg")
shift
;;
esac
done
found_switches=()
other_args=()
for arg in "$@"; do
for switch in "${switches[@]}"; do
if [[ "$arg" = "$switch" ]]; then
found_switches+=("\"$arg\"")
continue 2
fi
done
other_args+=("\"$arg\"")
done
echo "${found_switches[*]} -- ${other_args[*]}"

323
cccl_upstream/ci/util/git_bisect.sh Executable file
View File

@@ -0,0 +1,323 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ci_dir/.."
usage() {
cat <<USAGE
Usage: $0 [--preset NAME | --configure-override CMD] [options]
Generic Options:
-h, --help Show this help and exit
Bisection Options:
--good-ref STR Good ref/sha/tag/branch. Defaults to latest release tag.
Accepts '-Nd' (e.g., '-14d') to mean 'origin/main as of N days ago'.
--bad-ref STR Bad ref/sha/tag/branch. Defaults to origin/main.
Accepts '-Nd' (e.g., '-14d') to mean 'origin/main as of N days ago'.
--summary-file PATH Markdown summary output path (optional)
No summary file will be generated if this is omitted.
Build / Test Options:
--preset NAME CMake preset
--cmake-options STR Extra options passed to CMake preset configure (optional)
--configure-override CMD Command to run for configuration instead of cmake preset
If set, --preset and --cmake-options will be ignored
--build-targets STR Space separated ninja build targets (optional)
If omitted, no targets will be built -- explicitly specify 'all' if needed.
--ctest-targets STR Space separated CTest -R regex patterns (optional)
If omitted, no tests will be run -- explicitly specify '.' to run all.
--lit-precompile-tests STR Space-separated libcudacxx lit test paths to precompile without execution (optional)
e.g. 'cuda/utility/basic_any.pass.cpp'
--lit-tests STR Space-separated libcudacxx lit test paths to execute (optional)
e.g. 'cuda/utility/basic_any.pass.cpp'
--custom-test-cmd CMD Custom command run after build and tests (optional)
--repeat N Re-run the build/test for passing commits N times (default: 1)
USAGE
}
start_timestamp=${SECONDS}
function elapsed_time {
local duration=$(( SECONDS - start_timestamp ))
local minutes=$(( duration / 60 ))
local seconds=$(( duration % 60 ))
printf "%dm%02ds" "$minutes" "$seconds"
}
GOOD_REF=""
BAD_REF=""
PRESET=""
BUILD_TARGETS=""
CTEST_TARGETS=""
LIT_PRECOMPILE_TESTS=""
LIT_TESTS=""
SUMMARY_FILE=""
CMAKE_OPTIONS=""
CONFIGURE_OVERRIDE=""
CUSTOM_TEST_CMD=""
REPEAT=1
# Basic arg parser (keep simple, no extras)
while [[ $# -gt 0 ]]; do
case "$1" in
-h|--help) usage; exit 0 ;;
--preset) PRESET="${2:-}"; shift 2 ;;
--good-ref) GOOD_REF="${2:-}"; shift 2 ;;
--bad-ref) BAD_REF="${2:-}"; shift 2 ;;
--build-targets) BUILD_TARGETS="${2:-}"; shift 2 ;;
--ctest-targets) CTEST_TARGETS="${2:-}"; shift 2 ;;
--lit-precompile-tests) LIT_PRECOMPILE_TESTS="${2:-}"; shift 2 ;;
--lit-tests) LIT_TESTS="${2:-}"; shift 2 ;;
--cmake-options) CMAKE_OPTIONS="${2:-}"; shift 2 ;;
--configure-override) CONFIGURE_OVERRIDE="${2:-}"; shift 2 ;;
--summary-file) SUMMARY_FILE="${2:-}"; shift 2 ;;
--custom-test-cmd) CUSTOM_TEST_CMD="${2:-}"; shift 2 ;;
--repeat) REPEAT="${2:-}"; shift 2 ;;
*) echo "Unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
if [[ -z "${PRESET}" && -z "${CONFIGURE_OVERRIDE}" ]]; then
echo "::error:: --preset or --configure-override is required" >&2
usage
exit 2
fi
if [[ -n "${CONFIGURE_OVERRIDE}" ]]; then
if [[ -n "${PRESET}" ]]; then
echo "::warning:: --preset ignored due to --configure-override" >&2
fi
if [[ -n "${CMAKE_OPTIONS}" ]]; then
echo "::warning:: --cmake-options ignored due to --configure-override" >&2
fi
fi
# Ensure the checkout has complete history and tags:
git fetch --unshallow > /dev/null 2>&1 || :
git fetch --tags > /dev/null 2>&1 || :
# Resolve good and bad refs
good_ref="${GOOD_REF}"
bad_ref="${BAD_REF}"
# Helper to resolve '-Nd' (N days ago on origin/main) to a SHA
_resolve_days_ago() {
local spec="$1"
local base_branch="origin/main"
local n="${spec#-}"
n="${n%d}"
if [[ -z "$n" || ! "$n" =~ ^[0-9]+$ ]]; then
return 1
fi
local when
when=$(date -u -d "$n days ago" '+%Y-%m-%d %H:%M:%S %z')
git rev-list -n 1 --before="$when" "$base_branch"
}
# Resolve good_ref
if [[ -z "$good_ref" ]]; then
good_ref=$(git tag --list 'v*' | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -n 1 || :)
echo "Good ref defaulted to last release: $good_ref"
fi
if [[ "$good_ref" =~ ^-[0-9]+d$ ]]; then
good_sha=$(_resolve_days_ago "$good_ref")
if [[ -z "$good_sha" ]]; then
echo "::error::Unable to resolve good_ref '$good_ref' to a commit on origin/main" >&2
exit 1
fi
echo "Resolved good_ref '$good_ref' to origin/main @ $good_sha"
else
if [[ -z "$good_ref" ]]; then
echo "::error::Unable to determine good ref" >&2
exit 1
fi
good_sha=$(git rev-parse "$good_ref")
fi
# Resolve bad_ref
if [[ -z "$bad_ref" ]]; then
bad_ref="origin/main"
echo "Bad ref defaulted to origin/main: $bad_ref"
fi
if [[ "$bad_ref" =~ ^-[0-9]+d$ ]]; then
bad_sha=$(_resolve_days_ago "$bad_ref")
if [[ -z "$bad_sha" ]]; then
echo "::error::Unable to resolve bad_ref '$bad_ref' to a commit on origin/main" >&2
exit 1
fi
echo "Resolved bad_ref '$bad_ref' to origin/main @ $bad_sha"
else
bad_sha=$(git rev-parse "$bad_ref")
fi
# Copy the build-and-test runner to a temp file so it remains available as HEAD changes:
tmp_runner="$(mktemp /tmp/build-and-test-XXXXXX.sh)"
cp "${ci_dir}/util/build_and_test_targets.sh" "${tmp_runner}"
chmod +x "${tmp_runner}"
# If --repeat > 1, wrap the runner to repeat successful runs to detect flakiness.
bisect_runner="${tmp_runner}"
if [[ "${REPEAT}" =~ ^[0-9]+$ ]] && [[ "${REPEAT}" -gt 1 ]]; then
tmp_repeat="$(mktemp /tmp/build-and-test-repeat-XXXXXX.sh)"
cat > "${tmp_repeat}" <<EOF
#!/usr/bin/env bash
set -euo pipefail
attempts=${REPEAT}
for ((i=1; i<=attempts; ++i)); do
echo "::group::✅ build_and_test attempt \${i}/${REPEAT}"
"${tmp_runner}" "\$@" || {
echo "Attempt \${i} failed; marking commit as bad."
exit 1
}
echo "::endgroup::"
done
exit 0
EOF
chmod +x "${tmp_repeat}"
bisect_runner="${tmp_repeat}"
echo "Repeating successful runs ${REPEAT} times to check for flakiness."
fi
# Writer that always prints to stdout and tees to file if provided
write_summary() {
if [[ -n "${SUMMARY_FILE}" ]]; then
tee -a "${SUMMARY_FILE}"
else
cat
fi
}
echo "Starting bisect with:"
echo " BAD_SHA: $bad_sha"
echo " GOOD_SHA: $good_sha"
echo "::group::⚙️ Starting git bisect"
(set -x; git bisect start "$bad_sha" "$good_sha")
echo "::endgroup::"
bisect_log="$(mktemp /tmp/git-bisect-log-XXXXXX.log)"
bisect_output="$(mktemp /tmp/git-bisect-output-XXXXXX.log)"
(
set -x
git bisect run "${bisect_runner}" \
--preset "${PRESET}" \
--build-targets "${BUILD_TARGETS}" \
--ctest-targets "${CTEST_TARGETS}" \
--lit-precompile-tests "${LIT_PRECOMPILE_TESTS}" \
--lit-tests "${LIT_TESTS}" \
--cmake-options "${CMAKE_OPTIONS}" \
--configure-override "${CONFIGURE_OVERRIDE}" \
--custom-test-cmd "${CUSTOM_TEST_CMD}" \
| tee "${bisect_output}" || :
git bisect log | tee "${bisect_log}" || :
git bisect reset || :
)
if grep -q " is the first bad commit" "${bisect_output}"; then
bad_commit=$(awk '/ is the first bad commit/ {print $1}' "${bisect_output}")
echo -e "\e[1;32mFound bad commit in $(elapsed_time): $bad_commit\e[0m"
found=true
else
echo -e "\e[1;31mNo bad commit found ($(elapsed_time)).\e[0m"
found=false
fi
function print_repro {
echo "### ♻️ Reproduction Steps"
echo
echo '```bash'
if [[ -n "${LAUNCH_ARGS:-}" ]]; then
echo " .devcontainer/launch.sh \\"
echo " ${LAUNCH_ARGS} \\"
echo " -- \\"
fi
echo " ./ci/util/build_and_test_targets.sh \\"
declare -a build_test_vars=(
PRESET
CMAKE_OPTIONS
CONFIGURE_OVERRIDE
BUILD_TARGETS
CTEST_TARGETS
LIT_PRECOMPILE_TESTS
LIT_TESTS
CUSTOM_TEST_CMD
)
# only print the above vars if they're non-empty.
first=true
for var in "${build_test_vars[@]}"; do
if [[ -n "${!var:-}" ]]; then
flag="--${var,,}"
flag=${flag//_/-} # replace _ with -
if ! $first; then
echo " \\" # Trailing "\" to escape newlines
fi
echo -n " ${flag} \"${!var}\""
first=false
fi
done
echo # Final newline for last argument
echo '```'
}
if [[ "${found}" == "true" ]]; then
commit_info=$(git log "$bad_commit" -1 --pretty=format:'%h %s')
pr_ref=$(echo "$commit_info" | grep -oE '#[0-9]+' | head -n1 || :)
(
echo "## 🔎 Bisect Result"
echo
echo "- Culprit Commit: $commit_info"
if [[ -n "$pr_ref" ]]; then
pr_num=${pr_ref#\#}
echo "- Culprit PR: https://github.com/NVIDIA/cccl/pull/$pr_num"
fi
echo "- Commit SHA: $bad_commit"
echo "- Commit URL: https://github.com/NVIDIA/cccl/commit/${bad_commit}"
if [[ -n "${GHA_LOG_URL:-}" ]]; then
echo "- Bisection Logs: [GHA Job](${GHA_LOG_URL})"
fi
if [[ -n "${STEP_SUMMARY_URL:-}" ]]; then
echo "- Bisection Summary: [GHA Report](${STEP_SUMMARY_URL})"
fi
echo
print_repro
echo
echo "### ℹ️ Commit Details"
echo
echo '```'
git show "$bad_commit" --stat
echo '```'
echo
echo "### 🪵 Bisect Log"
echo
echo '```'
cat "${bisect_log}"
echo '```'
) | write_summary
else
(
echo "## ‼️ Bisect Failed"
echo
echo "git bisect did not resolve to a single commit."
echo
if [[ -n "${GHA_LOG_URL:-}" ]]; then
echo "- Bisection Logs: [GHA Job](${GHA_LOG_URL})"
echo "- Bisection Summary: [GHA Report](${STEP_SUMMARY_URL})"
fi
echo
print_repro
echo
echo "### 🪵 Bisect Log"
echo
echo '```'
cat "${bisect_log}"
echo '```'
) | write_summary
exit 1
fi

31
cccl_upstream/ci/util/manifest.sh Executable file
View File

@@ -0,0 +1,31 @@
#!/usr/bin/env bash
# Do not enable `eo pipefail`
# read relies on a quirk of unterminated/unquoted heredoc to capture input.
# TODO: rewrite using printf if we can keep nice formatting.
set -u
path="$(realpath "$1")"
outfile="$2"
version="$3"
platform="$4"
read -r -d '' manifest << EOF
{
"schema-version": 1,
"componentName": "cccl",
"componentVersion": "$version",
"platform": "$platform"
}
EOF
read -r -d '' prog << 'EOF'
split(.,"\n")
| select(length > 2)
| [{(.[]) : {"type": "header"}}]
| add
| $manifest + { "files" : . }
EOF
find "$path" -wholename '*include/*' -type f -printf '%P\n' \
| jq -s --raw-input --argjson manifest "$manifest" "$prog" > "${outfile}"

310
cccl_upstream/ci/util/memmon.sh Executable file
View File

@@ -0,0 +1,310 @@
#!/usr/bin/env bash
set -euo pipefail
pid_file="/tmp/.memmon.pid"
log_threshold="2"
print_threshold="5"
poll_interval="5"
log_file="$PWD/memmon.log"
mode=""
usage() {
cat <<'USAGE'
Monitors running processes for high memory usage, logging and reporting peaks above specified thresholds.
Usage: memmon.sh (--start | --stop | --monitor | --help)
[--log-threshold <GB>] # Write to log file if process exceeds this memory (default 2 GB)
[--print-threshold <GB>] # Print to stdout if process exceeds this memory (default 5 GB)
[--poll <seconds>] # Poll interval for checking processes (default 5 seconds)
[--log-file <file>] # Log file path (default ./memmon.log)
Modes:
--start Start monitoring in the background (writes pid to /tmp/.memmon.pid)
--stop Stop monitoring (kills pid in /tmp/.memmon.pid)
--monitor Run monitoring in the foreground (for testing/debugging)
Example Session:
memmon.sh --start # Start monitoring
launch_memory_intensive_processes # Do work
memmon.sh --stop # Stop monitoring and write log
cat memmon.log # View log
USAGE
}
error() {
echo "memmon: $*" >&2
exit 1
}
error_usage() {
echo "memmon: $*" >&2
usage
exit 1
}
ensure_absolute_log() {
case "$log_file" in
/*) return ;;
*)
local dir
dir="$(cd "$(dirname "$log_file")" && pwd)"
local base
base="$(basename "$log_file")"
log_file="$dir/$base"
;;
esac
}
to_kib() {
awk -v gib="$1" 'BEGIN {printf "%.0f", gib * 1024 * 1024}'
}
format_gib() {
awk -v rss="$1" 'BEGIN {printf "%.3f", rss/1024/1024}'
}
format_threshold() {
awk -v val="$1" 'BEGIN {printf "%.3f", val + 0}'
}
declare -A MEMMON_MAX_RSS
declare -A MEMMON_CMD
declare -A MEMMON_TARGET
get_cmdline() {
local pid="$1"
local cmdline_file="/proc/$pid/cmdline"
local raw
# Attempt to read from /proc first for accuracy
if [[ -r "$cmdline_file" ]]; then
raw="$(tr '\0' ' ' <"$cmdline_file" 2>/dev/null)"
# Clean up whitespace
raw="${raw//$'\n'/ }"
raw="${raw//$'\r'/ }"
raw="${raw//$'\t'/ }"
while [[ "$raw" == *' ' ]]; do
raw="${raw% }"
done
if [[ -n "$raw" ]]; then
printf '%s' "$raw"
return 0
fi
fi
# Fallback to ps if /proc is unavailable
raw="$(ps -wwp "$pid" -o command= 2>/dev/null | head -n1)"
# Clean up whitespace
raw="${raw//$'\n'/ }"
raw="${raw//$'\r'/ }"
raw="${raw//$'\t'/ }"
if [[ -n "$raw" ]]; then
printf '%s' "$raw"
return 0
fi
printf '[command unavailable]'
}
extract_target() {
local cmd="$1"
# Try to locate the name of the cmake target:
if [[ "$cmd" =~ CMakeFiles/([^[:space:]]*)\.dir ]]; then
printf '%s' "${BASH_REMATCH[1]}"
else
printf '-'
fi
}
start_memmon() {
# Check if already running
if [[ -f "$pid_file" ]]; then
local existing_pid
existing_pid="$(<"$pid_file")"
if kill -0 "$existing_pid" 2>/dev/null; then
error "already running (pid $existing_pid)"
fi
rm -f "$pid_file"
fi
ensure_absolute_log
# Start monitoring in the background
"$0" --monitor \
--log-threshold "$log_threshold" \
--print-threshold "$print_threshold" \
--poll "$poll_interval" \
--log-file "$log_file" &
local child_pid=$!
echo "$child_pid" >"$pid_file"
echo "memmon started (pid $child_pid, log-threshold ${log_threshold}GB, print-threshold ${print_threshold}GB, log $log_file)"
}
stop_memmon() {
if [[ ! -f "$pid_file" ]]; then
error "not running"
fi
local running_pid
running_pid="$(<"$pid_file")"
if ! kill -0 "$running_pid" 2>/dev/null; then
rm -f "$pid_file"
error "not running"
fi
# Attempt graceful shutdown
kill "$running_pid" 2>/dev/null || true
for _ in {1..20}; do
if ! kill -0 "$running_pid" 2>/dev/null; then
break
fi
sleep 0.25
done
# Force kill if still running
if kill -0 "$running_pid" 2>/dev/null; then
kill -9 "$running_pid" 2>/dev/null || true
fi
# Wait for pid file to be removed
for _ in {1..20}; do
[[ ! -f "$pid_file" ]] && break
sleep 0.25
done
[[ -f "$pid_file" ]] && rm -f "$pid_file"
echo "memmon stopped"
}
monitor_mem() {
MEMMON_MAX_RSS=()
MEMMON_CMD=()
MEMMON_TARGET=()
ensure_absolute_log
local log_threshold_kib
log_threshold_kib="$(to_kib "$log_threshold")"
local print_threshold_kib
print_threshold_kib="$(to_kib "$print_threshold")"
local running=true
cleanup() {
trap - INT TERM EXIT
mkdir -p "$(dirname "$log_file")"
{
printf "peak-mem | PID | target | command-line\n"
if [[ ${#MEMMON_MAX_RSS[@]} -eq 0 ]]; then
printf "No processes exceeded %s GB\n" "$(format_threshold "$log_threshold")"
else
local tmp
tmp="$(mktemp)"
for pid in "${!MEMMON_MAX_RSS[@]}"; do
printf "%s\t%s\t%s\t%s\n" "${MEMMON_MAX_RSS[$pid]}" "$pid" "${MEMMON_TARGET[$pid]}" "${MEMMON_CMD[$pid]}" >>"$tmp"
done
sort -nr -k1,1 "$tmp" | while IFS=$'\t' read -r peak pid target cmd; do
local mem_gib
mem_gib="$(format_gib "$peak")"
printf "%s GB | %s | %s | %s\n" "$mem_gib" "$pid" "$target" "$cmd"
done
rm -f "$tmp"
fi
} >"$log_file"
echo "memmon log written to $log_file"
rm -f "$pid_file"
}
trap 'running=false' INT TERM
trap cleanup EXIT
while $running; do
while read -r pid rss; do
[[ -z "$pid" || -z "$rss" ]] && continue
[[ "$pid" =~ ^[0-9]+$ ]] || continue
[[ "$rss" =~ ^[0-9]+$ ]] || continue
if (( rss >= log_threshold_kib )); then
local current=${MEMMON_MAX_RSS[$pid]:-0}
if (( rss > current )); then
MEMMON_MAX_RSS[$pid]=$rss
MEMMON_CMD[$pid]=$(get_cmdline "$pid")
MEMMON_TARGET[$pid]=$(extract_target "${MEMMON_CMD[$pid]}")
if (( rss >= print_threshold_kib )); then
local mem_gib
mem_gib="$(format_gib "$rss")"
printf 'memmon: %s GB | %s | %s | %s\n' "$mem_gib" "$pid" "${MEMMON_TARGET[$pid]}" "${MEMMON_CMD[$pid]}"
fi
fi
fi
done < <(ps -eo pid=,rss=)
if ! sleep "$poll_interval"; then
break
fi
done
}
while [[ $# -gt 0 ]]; do
case "$1" in
--log-threshold)
[[ $# -lt 2 ]] && error "--log-threshold requires a value"
log_threshold="$2"
shift 2
;;
--print-threshold)
[[ $# -lt 2 ]] && error "--print-threshold requires a value"
print_threshold="$2"
shift 2
;;
--log-file)
[[ $# -lt 2 ]] && error "--log-file requires a value"
log_file="$2"
shift 2
;;
--poll)
[[ $# -lt 2 ]] && error "--poll requires a value"
poll_interval="$2"
shift 2
;;
--start)
[[ -n "$mode" ]] && error "Specify only one of --start or --stop"
mode="start"
shift
;;
--stop)
[[ -n "$mode" ]] && error "Specify only one of --start or --stop"
mode="stop"
shift
;;
--monitor)
mode="monitor"
shift
;;
--help|-h)
usage
exit 0
;;
*)
error_usage "Unknown option: $1"
;;
esac
done
if [[ -z "$mode" ]]; then
error_usage "Must specify one of --start or --stop or --monitor"
fi
case "$mode" in
start)
start_memmon
;;
stop)
stop_memmon
;;
monitor)
monitor_mem
;;
*)
error_usage "Unhandled mode: $mode"
;;
esac

139
cccl_upstream/ci/util/pre-commit/check_cub_test_macros.py Executable file
View File

@@ -0,0 +1,139 @@
#!/usr/bin/env python3
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
#
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
import re
import sys
RAW_TEST_MACROS = (
"C2H_TEST",
"C2H_TEST_LIST",
"C2H_TEST_WITH_FIXTURE",
"C2H_TEST_LIST_WITH_FIXTURE",
"TEST_CASE",
"TEST_CASE_METHOD",
"SCENARIO",
"SCENARIO_METHOD",
"TEMPLATE_TEST_CASE",
"TEMPLATE_TEST_CASE_SIG",
"TEMPLATE_TEST_CASE_METHOD",
"TEMPLATE_TEST_CASE_METHOD_SIG",
"TEMPLATE_PRODUCT_TEST_CASE",
"TEMPLATE_PRODUCT_TEST_CASE_SIG",
"TEMPLATE_PRODUCT_TEST_CASE_METHOD",
"TEMPLATE_PRODUCT_TEST_CASE_METHOD_SIG",
"TEMPLATE_LIST_TEST_CASE",
"TEMPLATE_LIST_TEST_CASE_METHOD",
)
RAW_TEST_MACRO_RE = re.compile(
r"^[ \t]*(?P<macro>"
+ "|".join(
re.escape(macro) for macro in sorted(RAW_TEST_MACROS, key=len, reverse=True)
)
+ r")[ \t]*\(",
re.MULTILINE,
)
def remove_comments(source: str) -> str:
"""Blank C++ comments while preserving line and column positions."""
result = list(source)
index = 0
while index < len(source):
if source.startswith("//", index):
end = source.find("\n", index)
if end == -1:
end = len(source)
for comment_index in range(index, end):
result[comment_index] = " "
index = end
continue
if source.startswith("/*", index):
end = source.find("*/", index + 2)
if end == -1:
end = len(source) - 2
for comment_index in range(index, min(end + 2, len(source))):
if source[comment_index] not in "\r\n":
result[comment_index] = " "
index = end + 2
continue
if source[index] in {'"', "'"}:
quote = source[index]
index += 1
while index < len(source):
if source[index] == "\\":
index += 2
continue
if source[index] == quote:
index += 1
break
index += 1
continue
index += 1
return "".join(result)
def self_test() -> bool:
fixtures = [
('// C2H_TEST("x")', False),
('/*\nTEST_CASE("x")\n*/', False),
('const char* value = "TEST_CASE(";', False),
('CUB_TEST("x", "[y]", CUB_SMALL)', False),
('CUB_TEST_CASE("x", "[y]", CUB_LARGE)', False),
('CUB_TEST_LIST("x", "[y]", CUB_SMALL, types)', False),
]
fixtures.extend((f'{macro}("x")', True) for macro in RAW_TEST_MACROS)
for source, expected in fixtures:
found = bool(RAW_TEST_MACRO_RE.search(remove_comments(source)))
if found != expected:
expected_result = "match" if expected else "no match"
actual_result = "match" if found else "no match"
print(
"internal error: test-registration checker self-test failed for "
f"{source!r}: expected {expected_result}, found {actual_result}. "
"This is a problem with the checker, not the files being committed.",
file=sys.stderr,
)
return False
return True
def check_file(filename: str) -> bool:
with open(filename, encoding="utf-8", errors="surrogateescape") as source_file:
source = source_file.read()
source_without_comments = remove_comments(source)
found_error = False
for match in RAW_TEST_MACRO_RE.finditer(source_without_comments):
line = source.count("\n", 0, match.start()) + 1
column = match.start("macro") - source.rfind("\n", 0, match.start("macro"))
print(
f"{filename}:{line}:{column}: {match.group('macro')} bypasses CUB "
"memory classification; use CUB_TEST, CUB_TEST_CASE, or CUB_TEST_LIST."
)
found_error = True
return found_error
def main() -> int:
if not self_test():
return 2
found_error = False
for filename in sys.argv[1:]:
found_error = check_file(filename) or found_error
return int(found_error)
if __name__ == "__main__":
sys.exit(main())

53
cccl_upstream/ci/util/pre-commit/check_shebang.py Executable file
View File

@@ -0,0 +1,53 @@
#!/usr/bin/env python3
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
#
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
import re
import sys
def main() -> int:
# We pre-compile this regular expression as a micro-optimization for speed. The
# assumption is that the shebang is correct, so everything in the happy path should be
# as fast as possible.
FIRST_LINE_RE = re.compile(r"#!\s*/usr/bin/env.*")
ret = 0
for f in sys.argv[1:]:
with open(f) as fd:
first_line = fd.readline()
if not first_line.startswith("#!"):
# Not a shebang
continue
if FIRST_LINE_RE.match(first_line):
# Already correct
continue
ret = 1
if not (
m := re.match(r"#!\s*(?:/bin/(\w+)|/usr/bin/(\w+))\s*(.*)", first_line)
):
# Not assert, pre-commit may compile with -O
raise AssertionError(f"Failed to match shebang for {first_line}")
fixed = f"#!/usr/bin/env {m[1] or m[2]}".rstrip()
if rest := m[3].strip():
fixed += f" {rest}"
fixed += "\n"
with open(f) as fd:
# Read the remaining lines, we need them in order to overwrite
lines = fd.readlines()
lines[0] = fixed
with open(f, "w") as fd:
fd.writelines(lines)
return ret
if __name__ == "__main__":
sys.exit(main())

94
cccl_upstream/ci/util/pre-commit/strip_unprintable.py Executable file
View File

@@ -0,0 +1,94 @@
#!/usr/bin/env python3
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
#
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# strip_unprintable.py - Remove invisible / unprintable characters from text files.
import re
import sys
from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter
# Canonical definition of what gets removed. One row per range group:
# (regex character-class fragment, human description)
# The compiled character class and the --help listing are both derived from this
# table, so adding or removing a range only needs to happen here. TAB (U+0009),
# LF (U+000A), and CR (U+000D) are deliberately excluded from the C0 range. The
# fragments use \x/\u escapes so the source file itself stays free of the very
# characters this script removes.
RANGES = (
(
r"\x00-\x08\x0b\x0c\x0e-\x1f\x7f",
"C0 controls / DEL (TAB, LF, CR preserved)",
),
(r"\x80-\x9f", "C1 controls"),
(r"\xa0", "no-break space"),
(r"\u200b-\u200f", "zero-width space/joiners, bidi marks"),
(r"\u202a-\u202e", "bidi embedding/override"),
(r"\u2060-\u2064", "word joiner, invisible operators"),
(r"\ufeff", "BOM / zero-width no-break space"),
)
# Character class assembled from column 1 of the ranges table.
BAD_RE = re.compile("[" + "".join(frag for frag, _ in RANGES) + "]")
def parse_args() -> Namespace:
removed = "\n".join(f" {frag}\t{desc}" for frag, desc in RANGES)
parser = ArgumentParser(
description=(
"Remove invisible / unprintable characters from text files, in place, "
"while preserving ordinary whitespace (TAB U+0009, LF U+000A, "
"CR U+000D)."
),
epilog=f"Removed characters:\n{removed}",
formatter_class=RawDescriptionHelpFormatter,
)
parser.add_argument(
"--check",
action="store_true",
help=(
"Report offending files and their line/column locations; make no "
"edits. Exits non-zero if any unprintable characters are found."
),
)
parser.add_argument("files", nargs="+", metavar="FILE")
return parser.parse_args()
def check(files: list[str]) -> int:
ret = 0
for f in files:
with open(f, encoding="utf-8", errors="surrogateescape") as fd:
for lineno, line in enumerate(fd, start=1):
for m in BAD_RE.finditer(line):
print(f"{f}:{lineno}:{m.start() + 1}: U+{ord(m.group()):04X}")
ret = 1
return ret
def strip(files: list[str]) -> int:
ret = 0
for f in files:
with open(f, encoding="utf-8", errors="surrogateescape") as fd:
original = fd.read()
stripped = BAD_RE.sub("", original)
if stripped != original:
with open(
f, "w", encoding="utf-8", errors="surrogateescape", newline=""
) as fd:
fd.write(stripped)
ret = 1
return ret
def main() -> int:
args = parse_args()
if args.check:
return check(args.files)
return strip(args.files)
if __name__ == "__main__":
sys.exit(main())

58
cccl_upstream/ci/util/python/common_arg_parser.sh Normal file
View File

@@ -0,0 +1,58 @@
# Argument parser for Python CI scripts.
parse_python_args() {
# Initialize variables
py_version=""
# ctk_mode carries the -ctk-mode value; empty means the default ("pinned").
ctk_mode=""
while [[ $# -gt 0 ]]; do
case $1 in
-py-version=*)
py_version="${1#*=}"
shift
;;
-py-version)
if [[ $# -lt 2 ]]; then
echo "Error: -py-version requires a value" >&2
return 1
fi
py_version="$2"
shift 2
;;
-ctk-mode=*)
ctk_mode="${1#*=}"
# Reject an explicit-but-empty value (e.g. `-ctk-mode=`): a lane
# that wants the default omits the flag entirely, so an empty
# value signals a malformed generated argument -- fail loudly.
if [[ -z "${ctk_mode}" ]]; then
echo "Error: -ctk-mode requires a value" >&2
return 1
fi
shift
;;
-ctk-mode)
if [[ $# -lt 2 || -z "$2" ]]; then
echo "Error: -ctk-mode requires a value" >&2
return 1
fi
ctk_mode="$2"
shift 2
;;
*)
# Unknown argument, ignore
shift
;;
esac
done
# Export for use by the calling script (py_version and ctk_mode are its inputs).
export py_version ctk_mode
}
require_py_version() {
if [[ -z "$py_version" ]]; then
echo "Error: -py-version is required" >&2
[[ -n "$1" ]] && echo "$1" >&2
return 1
fi
}

32
cccl_upstream/ci/util/retry.sh Executable file
View File

@@ -0,0 +1,32 @@
#!/usr/bin/env bash
if [[ "$#" -lt 3 ]]; then
echo "Usage: $0 num_tries sleep_time command [args...]"
echo " num_tries: Number of attempts to run the command"
echo " sleep_time: Time to wait between attempts (in seconds)"
echo " command: The command to run"
echo " args: Arguments to pass to the command"
exit 1
fi
num_tries=$1
sleep_time=$2
shift 2
command=("${*@Q}")
# Loop until the command succeeds or we reach the maximum number of attempts:
for ((i=1; i<=num_tries; i++)); do
echo "Attempt ${i} of ${num_tries}: Running command '${command[*]}'"
status=0
eval "${command[*]}" || status=$?
if [[ "$status" -eq 0 ]]; then
echo "Command '${command[*]}' succeeded on attempt ${i}."
exit 0
else
echo "Command '${command[*]}' failed with status ${status}. Retrying in ${sleep_time} seconds..."
sleep "$sleep_time"
fi
done
echo "Command '${command[*]}' failed after ${num_tries} attempts."
exit 1

85
cccl_upstream/ci/util/version_compare.sh Executable file
View File

@@ -0,0 +1,85 @@
#!/usr/bin/env bash
set -euo pipefail
usage=$(cat <<EOF
Usage: $0 X.Y[.Z[.W[...]]] <compare_op> A.B[.C[.D[...]]]
Compares two version strings with the specified operator.
compare_ops:
lt - less than
le - less than or equal to
eq - equal to
ne - not equal to
ge - greater than or equal to
gt - greater than
EOF
)
readonly usage
if [[ "$#" -ne 3 ]]; then
echo "Error: Invalid arguments: $*" >&2
echo "$usage" >&2
exit 1
fi
version_a="$1"
operator="$2"
version_b="$3"
# Validate operator
if [[ ! "$operator" =~ ^(lt|le|eq|ne|ge|gt)$ ]]; then
echo "Error: Invalid operator '$operator'. Must be one of: lt, le, eq, ne, ge, gt." >&2
echo "$usage" >&2
exit 1
fi
# Validate versions:
version_regex='^[0-9]+(\.[0-9]+)*$'
if [[ ! "$version_a" =~ $version_regex ]]; then
echo "Error: Invalid version string '$version_a'." >&2
echo "$usage" >&2
exit 1
fi
if [[ ! "$version_b" =~ $version_regex ]]; then
echo "Error: Invalid version string '$version_b'." >&2
echo "$usage" >&2
exit 1
fi
# Split versions into arrays
IFS='.' read -r -a ver_a_parts <<< "$version_a"
IFS='.' read -r -a ver_b_parts <<< "$version_b"
max_length=${#ver_a_parts[@]}
if [[ "${#ver_b_parts[@]}" -gt "$max_length" ]]; then
max_length=${#ver_b_parts[@]}
fi
# Compare each part
for ((i=0; i<max_length; i++)); do
part_a=${ver_a_parts[i]:-0}
part_b=${ver_b_parts[i]:-0}
if ((part_a < part_b)); then
result="lt"
break
elif ((part_a > part_b)); then
result="gt"
break
else
result="eq"
fi
done
# Evaluate the comparison based on the operator
case "$operator" in
lt) [[ "$result" == "lt" ]] ;;
le) [[ "$result" == "lt" || "$result" == "eq" ]] ;;
eq) [[ "$result" == "eq" ]] ;;
ne) [[ "$result" != "eq" ]] ;;
ge) [[ "$result" == "gt" || "$result" == "eq" ]] ;;
gt) [[ "$result" == "gt" ]] ;;
*) echo "Error: Unhandled operator '${operator}'." >&2; exit 1 ;;
esac
exit $?

34
cccl_upstream/ci/util/workflow/common.sh Executable file
View File

@@ -0,0 +1,34 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
echo "This script must be sourced, not executed directly." >&2
exit 1
fi
if [[ -z "${GITHUB_ACTIONS:-}" ]]; then
echo "This script must be run in a GitHub Actions environment." >&2
exit 1
fi
to_posix_path() {
local path="$1"
if [[ "$path" =~ ^([A-Za-z]):([\\/]?.*)$ ]]; then
local drive="${BASH_REMATCH[1]}"
local rest="${BASH_REMATCH[2]}"
rest="${rest//\\/\/}"
printf '/%s%s\n' "${drive,,}" "$rest"
return
fi
printf '%s\n' "$path"
}
runner_temp_posix="$(to_posix_path "${RUNNER_TEMP:-/tmp}")"
export WORKFLOW_ARTIFACT="workflow"
export WORKFLOW_DIR="${runner_temp_posix}/workflow"
mkdir -p "$WORKFLOW_DIR"

43
cccl_upstream/ci/util/workflow/get_consumers.sh Executable file
View File

@@ -0,0 +1,43 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/workflow/common.sh
source "$ci_dir/util/workflow/common.sh"
usage=$(cat <<EOF
Usage: $0 [job_id]
Return a json array of job definitions for all consumers of the specified producer job ID.
If no job ID is provided, the \$JOB_ID environment variable is used.
EOF
)
readonly usage
if [[ "$#" -gt 1 ]]; then
echo "Error: Too many arguments." >&2
echo "$usage" >&2
exit 1
fi
job_id="${1:-${JOB_ID:-}}"
if [[ -z "$job_id" ]]; then
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
echo "$usage" >&2
exit 1
fi
"${ci_dir}/util/workflow/initialize.sh"
consumers=$(jq --arg job_id "$job_id" '
to_entries[]
| select(.value.two_stage)
| .value.two_stage[]
| select(any(.producers[]; .id == $job_id))
| .consumers
' "$WORKFLOW_DIR/workflow.json")
echo "$consumers"

51
cccl_upstream/ci/util/workflow/get_job_def.sh Executable file
View File

@@ -0,0 +1,51 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/workflow/common.sh
source "$ci_dir/util/workflow/common.sh"
usage=$(cat <<EOF
Usage: $0 [job_id]
Prints a json object containing the workflow job definition for the specified job ID.
If no job ID is provided, the \$JOB_ID environment variable is used.
If the job ID does not exist in the workflow, an error is raised.
EOF
)
readonly usage
if [[ "$#" -gt 1 ]]; then
echo "Error: Too many arguments." >&2
echo "$usage" >&2
exit 1
fi
job_id="${1:-${JOB_ID:-}}"
if [[ -z "$job_id" ]]; then
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
echo "$usage" >&2
exit 1
fi
"${ci_dir}/util/workflow/initialize.sh"
job_obj=$(jq --arg job_id "$job_id" '
to_entries[]
| .value
| (
(select(has("standalone")) | .standalone[] | select(.id == $job_id)) //
(select(has("two_stage")) | .two_stage[] | .producers[] | select(.id == $job_id)) //
(select(has("two_stage")) | .two_stage[] | .consumers[] | select(.id == $job_id))
)
' "$WORKFLOW_DIR/workflow.json")
if [[ -z "$job_obj" ]]; then
echo "Error: No job definition found for job ID '$job_id'." >&2
exit 1
fi
echo "$job_obj" | jq -r

34
cccl_upstream/ci/util/workflow/get_job_project.sh Executable file
View File

@@ -0,0 +1,34 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
usage=$(cat <<EOF
Usage: $0 [job_id]
Returns the name of the project built by the specified job id.
If no job ID is provided, the \$JOB_ID environment variable is used.
If the job ID does not exist in the workflow an error is raised.
EOF
)
readonly usage
if [[ "$#" -gt 1 ]]; then
echo "Error: Too many arguments." >&2
echo "$usage" >&2
exit 1
fi
job_id="${1:-${JOB_ID:-}}"
if [[ -z "$job_id" ]]; then
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
echo "$usage" >&2
exit 1
fi
job_def=$("${ci_dir}/util/workflow/get_job_def.sh" "$job_id")
project=$(echo "$job_def" | jq -r '.origin.matrix_job.project')
echo "$project"

55
cccl_upstream/ci/util/workflow/get_producer_id.sh Executable file
View File

@@ -0,0 +1,55 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/workflow/common.sh
source "$ci_dir/util/workflow/common.sh"
usage=$(cat <<EOF
Usage: $0 [job_id]
Prints the job ID of the associated producer for the specified consumer job ID.
If no job ID is provided, the \$JOB_ID environment variable is used.
If the number of producers for the job is not exactly one, an error is raised.
EOF
)
readonly usage
if [[ "$#" -gt 1 ]]; then
echo "Error: Too many arguments." >&2
echo "$usage" >&2
exit 1
fi
job_id="${1:-${JOB_ID:-}}"
if [[ -z "$job_id" ]]; then
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
echo "$usage" >&2
exit 1
fi
"${ci_dir}/util/workflow/initialize.sh"
producers=$(jq --arg job_id "$job_id" '
to_entries[]
| select(.value.two_stage)
| .value.two_stage[]
| select(any(.consumers[]; .id == $job_id))
| .producers
' "$WORKFLOW_DIR/workflow.json")
producer_count=$(echo "$producers" | jq 'length')
if [[ "$producer_count" -ne 1 ]]; then
echo "Error: Expected exactly one producer for job ID '$job_id', but found ${producer_count:-0}." >&2
exit 1
fi
producer_id=$(echo "$producers" | jq -r '.[0].id')
if [[ -z "$producer_id" ]]; then
echo "Error: No producer ID found for job ID '$job_id'." >&2
exit 1
fi
echo "$producer_id"

43
cccl_upstream/ci/util/workflow/get_producers.sh Executable file
View File

@@ -0,0 +1,43 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/workflow/common.sh
source "$ci_dir/util/workflow/common.sh"
usage=$(cat <<EOF
Usage: $0 [job_id]
Return a json array of job definitions for all producers of the specified consumer job ID.
If no job ID is provided, the \$JOB_ID environment variable is used.
EOF
)
readonly usage
if [[ "$#" -gt 1 ]]; then
echo "Error: Too many arguments." >&2
echo "$usage" >&2
exit 1
fi
job_id="${1:-${JOB_ID:-}}"
if [[ -z "$job_id" ]]; then
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
echo "$usage" >&2
exit 1
fi
"${ci_dir}/util/workflow/initialize.sh"
producers=$(jq --arg job_id "$job_id" '
to_entries[]
| select(.value.two_stage)
| .value.two_stage[]
| select(any(.consumers[]; .id == $job_id))
| .producers
' "$WORKFLOW_DIR/workflow.json")
echo "$producers"

35
cccl_upstream/ci/util/workflow/get_stable_job_hash.sh Executable file
View File

@@ -0,0 +1,35 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
usage=$(cat <<EOF
Usage: $0 [job_id]
Get a stable hash that identifies the job's toolchain, runner, image,
name, and launch command, removing origin and per-run ids.
If no job ID is provided, the \$JOB_ID environment variable is used.
If the job ID does not exist in the workflow an error is raised.
EOF
)
readonly usage
if [[ "$#" -gt 1 ]]; then
echo "Error: Too many arguments." >&2
echo "$usage" >&2
exit 1
fi
job_id="${1:-${JOB_ID:-}}"
if [[ -z "$job_id" ]]; then
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
echo "$usage" >&2
exit 1
fi
job_def=$("${ci_dir}/util/workflow/get_job_def.sh" "$job_id" | jq 'del(.id, .origin)')
job_hash=$(echo "$job_def" | sha256sum | awk '{print $1}')
echo "$job_hash"

68
cccl_upstream/ci/util/workflow/get_wheel_artifact_name.sh Executable file
View File

@@ -0,0 +1,68 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
usage=$(cat <<EOF
Usage: $0 [job_id]
Get the name of the wheel file that matches the specified job ID's configuration.
If no job ID is provided, the \$JOB_ID environment variable is used.
If the job ID does not exist in the workflow, or is not a python job, an error is raised.
EOF
)
readonly usage
if [[ "$#" -gt 1 ]]; then
echo "Error: Too many arguments." >&2
echo "$usage" >&2
exit 1
fi
job_id="${1:-${JOB_ID:-}}"
if [[ -z "$job_id" ]]; then
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
echo "$usage" >&2
exit 1
fi
job_def=$("${ci_dir}/util/workflow/get_job_def.sh" "$job_id")
py_version=$(echo "$job_def" | jq -r '.origin.matrix_job.py_version')
host=$(echo "$job_def" | jq -r '.origin.matrix_job.cxx_family')
if [[ "$host" == "MSVC" ]]; then
os="windows"
else
os="linux"
fi
arch=$(echo "$job_def" | jq -r '.origin.matrix_job.cpu')
project=$(echo "$job_def" | jq -r '.origin.matrix_job.project')
for tag in "$py_version" "$os" "$arch"; do
if [[ -z "$tag" ]]; then
echo "Error: Missing required field in job definition for job ID '$job_id'." >&2
echo "$usage" >&2
echo >&2
"Job definition: $job_def" >&2
exit 1
fi
done
# v1 and v2 Python build jobs both run in the same workflow, so their wheel
# artifacts must have distinct names or the second upload clobbers the first
# and downstream test jobs grab the wrong wheel. v1 keeps its historical name
# (the test-cpu-import workflow hardcodes it); v2 gets a "-v2" suffix.
suffix=""
if [[ "$project" == "python_v2" ]]; then
suffix="-v2"
elif [[ "$project" == "python_tsan" ]]; then
# ThreadSanitizer-instrumented wheel (free-threaded TSan nightly lane). Must
# be distinct so its build doesn't clobber the normal wheel and the TSan test
# job doesn't grab an uninstrumented one.
suffix="-tsan"
fi
echo "wheel-cccl${suffix}-$os-$arch-py$py_version"

47
cccl_upstream/ci/util/workflow/has_consumers.sh Executable file
View File

@@ -0,0 +1,47 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/workflow/common.sh
source "$ci_dir/util/workflow/common.sh"
usage=$(cat <<EOF
Usage: $0 [job_id]
Exits successfully if the specified job ID has consumers, otherwise exits with an error.
If no job ID is provided, the \$JOB_ID environment variable is used.
EOF
)
readonly usage
if [[ "$#" -gt 1 ]]; then
echo "Error: Too many arguments." >&2
echo "$usage" >&2
exit 1
fi
job_id="${1:-${JOB_ID:-}}"
if [[ -z "$job_id" ]]; then
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
echo "$usage" >&2
exit 1
fi
"${ci_dir}/util/workflow/initialize.sh"
matching_producer=$(jq --arg job_id "$job_id" '
to_entries[]
| select(.value.two_stage)
| .value.two_stage[]
| .producers[]
| select(.id == $job_id)
' "$WORKFLOW_DIR/workflow.json")
if [[ -n "$matching_producer" ]]; then
exit 0
else
exit 1
fi

47
cccl_upstream/ci/util/workflow/has_producers.sh Executable file
View File

@@ -0,0 +1,47 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/workflow/common.sh
source "$ci_dir/util/workflow/common.sh"
usage=$(cat <<EOF
Usage: $0 [job_id]
Exits successfully if the specified job ID has producers, otherwise exits with an error.
If no job ID is provided, the \$JOB_ID environment variable is used.
EOF
)
readonly usage
if [[ "$#" -gt 1 ]]; then
echo "Error: Too many arguments." >&2
echo "$usage" >&2
exit 1
fi
job_id="${1:-${JOB_ID:-}}"
if [[ -z "$job_id" ]]; then
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
echo "$usage" >&2
exit 1
fi
"${ci_dir}/util/workflow/initialize.sh"
matching_consumer=$(jq --arg job_id "$job_id" '
to_entries[]
| select(.value.two_stage)
| .value.two_stage[]
| .consumers[]
| select(.id == $job_id)
' "$WORKFLOW_DIR/workflow.json")
if [[ -n "$matching_consumer" ]]; then
exit 0
else
exit 1
fi

26
cccl_upstream/ci/util/workflow/initialize.sh Executable file
View File

@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -euo pipefail
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
readonly ci_dir
# shellcheck source=ci/util/workflow/common.sh
source "$ci_dir/util/workflow/common.sh"
usage=$(cat <<EOF
Usage: $0
Downloads the workflow artifact and unpacks it to \$WORKFLOW_DIR, but only if it doesn't already exist.
EOF
)
readonly usage
if [[ "$#" -ne 0 ]]; then
echo "Error: This script does not take any arguments." >&2
echo "$usage" >&2
exit 1
fi
if [[ ! -f "$WORKFLOW_DIR/workflow.json" ]]; then
"$ci_dir/util/artifacts/download/fetch.sh" "$WORKFLOW_ARTIFACT" "$WORKFLOW_DIR" > /dev/null
fi