feat(cccl): integrate missing CCCL directories — python/, ci/, .agent/, docs/, test/
Sparse-checkout from NVIDIA/cccl main branch to complete cccl_upstream: Added: - python/cuda_cccl/ (226 files) — Python bindings for device-level algorithms Critical for muh toolchain: cuda.compute.reduce_into, scan, radix_sort, etc. Includes 204 .py files with full test coverage for all 27 algorithms - ci/ (163 files) — Build/test infrastructure build_cub.sh, test_cub.sh, build_and_test_targets.sh, matrix.yaml Directly maps to our [INFRA-CI] and [INFRA-BUILD] items - .agent/skills/ (7 files) — NVIDIA's own agent skills for CCCL cccl-style/SKILL.md, cccl-test/SKILL.md, sass-diff/SKILL.md - docs/ (491 files) — Official CCCL documentation CI references, CMake guides, Python compute docs, libcudacxx PTX docs - test/ (12 files) — Top-level integration tests (cuda_smoke, stdpar) - Root configs: .clang-format, .clang-tidy, CONTRIBUTING.md, pyproject.toml - CLAUDE.md symlink → AGENTS.md (NVIDIA's standard) cccl_upstream now mirrors full NVIDIA/cccl structure: Before: 42M (cub + thrust + libcudacxx + cudax + c + examples + benchmarks) After: 53M (+python +ci +docs +.agent +test +configs) This completes the CCCL base needed for: - [muh-bench] items: ci/util/build_and_test_targets.sh for targeted builds - [CCCL-verify] items: python/cuda_cccl/tests/ as reference implementations - [CCCL-test] items: ci/test_cub.sh, ci/test_thrust.sh - Agent workflow: .agent/skills/ for consistent style and test patterns
This commit is contained in:
39
cccl_upstream/ci/util/artifacts/common.sh
Executable file
39
cccl_upstream/ci/util/artifacts/common.sh
Executable file
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
||||
echo "This script must be sourced, not executed directly." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${GITHUB_ACTIONS:-}" ]]; then
|
||||
echo "This script must be run in a GitHub Actions environment." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
to_posix_path() {
|
||||
local path="$1"
|
||||
|
||||
if [[ "$path" =~ ^([A-Za-z]):([\\/]?.*)$ ]]; then
|
||||
local drive="${BASH_REMATCH[1]}"
|
||||
local rest="${BASH_REMATCH[2]}"
|
||||
rest="${rest//\\/\/}"
|
||||
printf '/%s%s\n' "${drive,,}" "$rest"
|
||||
return
|
||||
fi
|
||||
|
||||
printf '%s\n' "$path"
|
||||
}
|
||||
|
||||
runner_temp_posix="$(to_posix_path "${RUNNER_TEMP:-/tmp}")"
|
||||
|
||||
export ARTIFACT_UPLOAD_STAGE="${runner_temp_posix}/artifact_upload_stage"
|
||||
export ARTIFACT_ARCHIVES="${runner_temp_posix}/artifact_archives"
|
||||
export ARTIFACT_UPLOAD_REGISTERY="${ARTIFACT_UPLOAD_STAGE}/artifact_upload_registry.json"
|
||||
|
||||
mkdir -p "$ARTIFACT_UPLOAD_STAGE" "$ARTIFACT_ARCHIVES"
|
||||
|
||||
if [[ ! -f "$ARTIFACT_UPLOAD_REGISTERY" ]]; then
|
||||
echo "[]" > "$ARTIFACT_UPLOAD_REGISTERY"
|
||||
fi
|
||||
40
cccl_upstream/ci/util/artifacts/download.sh
Executable file
40
cccl_upstream/ci/util/artifacts/download.sh
Executable file
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <name> [<path>]
|
||||
|
||||
Download artifacts uploaded by other jobs in this CI run.
|
||||
|
||||
Example Usage:
|
||||
Download an artifact to the current directory:
|
||||
$0 source_artifact.tar.gz
|
||||
|
||||
Download a packed artifact and extract it to the provided path:
|
||||
$0 job-\$ID-products some/path/
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -lt 1 ]]; then
|
||||
echo "Error: Missing artifact name." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
readonly artifact_name="$1"
|
||||
|
||||
if [[ "$#" -eq 1 ]]; then
|
||||
artifact_path="./"
|
||||
else
|
||||
artifact_path="$2"
|
||||
fi
|
||||
|
||||
start=$SECONDS
|
||||
"$ci_dir/util/artifacts/download/fetch.sh" "$artifact_name" "$artifact_path"
|
||||
echo "Artifact '$artifact_name' downloaded to '$artifact_path' in $((SECONDS - start)) seconds."
|
||||
40
cccl_upstream/ci/util/artifacts/download/fetch.sh
Executable file
40
cccl_upstream/ci/util/artifacts/download/fetch.sh
Executable file
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <artifact_name> <target_directory>
|
||||
|
||||
Downloads files from a named artifact from the current CI workflow run into the specified directory.
|
||||
|
||||
Example Usages:
|
||||
- $0 my_artifact.tar.gz ./
|
||||
- $0 my_artifact /path/to/some/directory/
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -lt 2 ]]; then
|
||||
echo "Error: Missing arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
readonly artifact_name="$1"
|
||||
|
||||
# Create the target directory and then get its absolute path
|
||||
mkdir -p "$2"
|
||||
target_directory="$(cd "$2" && pwd)"
|
||||
readonly target_directory
|
||||
|
||||
echo "Downloading artifact '$artifact_name' to '$target_directory'"
|
||||
# shellcheck disable=SC2154
|
||||
"$ci_dir/util/retry.sh" 5 30 \
|
||||
gh run download "${GITHUB_RUN_ID}" \
|
||||
--name "$artifact_name" \
|
||||
--dir "$target_directory"
|
||||
45
cccl_upstream/ci/util/artifacts/download/unpack.sh
Executable file
45
cccl_upstream/ci/util/artifacts/download/unpack.sh
Executable file
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <artifact_name> <artifact_path>
|
||||
|
||||
Unpacks a fetched packed artifact's tar.zst archive into the specified directory.
|
||||
|
||||
Example Usages:
|
||||
- $0 /tmp/my_artifact.tar.zst /tmp/my_artifact
|
||||
- $0 /path/to/archive.tar.zst /path/to/extract/
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -lt 2 ]]; then
|
||||
echo "Error: Missing arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v zstd > /dev/null 2>&1; then
|
||||
echo "Error: zstd not found." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
readonly artifact_name="$1"
|
||||
readonly artifact_path="$2"
|
||||
|
||||
readonly artifact_archive="$ARTIFACT_ARCHIVES/${artifact_name}.tar.zst"
|
||||
|
||||
echo "Unpacking artifact from '$artifact_archive' to '$artifact_path'"
|
||||
echo "Using zstd executable: $(command -v zstd)"
|
||||
|
||||
# Create the artifact path directory if it doesn't exist
|
||||
mkdir -p "$artifact_path"
|
||||
|
||||
zstd --decompress --threads=0 --stdout "$artifact_archive" \
|
||||
| tar -xv -C "$artifact_path"
|
||||
45
cccl_upstream/ci/util/artifacts/download_packed.sh
Executable file
45
cccl_upstream/ci/util/artifacts/download_packed.sh
Executable file
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <name> [<path>]
|
||||
|
||||
Download and extracts a packed artifact uploaded by another job in this CI run.
|
||||
|
||||
Example Usage:
|
||||
Download an artifact to the current directory:
|
||||
$0 source_artifact.tar.gz
|
||||
|
||||
Download a packed artifact and extract it to the provided path:
|
||||
$0 job-\$ID-products build/
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -lt 1 ]]; then
|
||||
echo "Error: Missing artifact name." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
readonly artifact_name="$1"
|
||||
|
||||
if [[ "$#" -eq 1 ]]; then
|
||||
artifact_path="./"
|
||||
else
|
||||
artifact_path="$2"
|
||||
fi
|
||||
|
||||
start=$SECONDS
|
||||
"$ci_dir/util/artifacts/download/fetch.sh" "$artifact_name" "${ARTIFACT_ARCHIVES}"
|
||||
fetched=$SECONDS
|
||||
"$ci_dir/util/artifacts/download/unpack.sh" "$artifact_name" "$artifact_path"
|
||||
unpacked=$SECONDS
|
||||
|
||||
echo "Artifact '$artifact_name' fetched in $((fetched - start)) seconds."
|
||||
echo "Artifact '$artifact_name' unpacked in $((unpacked - fetched)) seconds."
|
||||
70
cccl_upstream/ci/util/artifacts/stage.sh
Executable file
70
cccl_upstream/ci/util/artifacts/stage.sh
Executable file
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <artifact_name> <regex> [<regex> ...]
|
||||
|
||||
Stages files matching the provided regexes path for upload under the specified artifact.
|
||||
Regexes are passed to the 'find' command's -regex option within the artifact stage path and implicitly
|
||||
start with '^\\./'.
|
||||
|
||||
Staged files can be unstaged using the 'artifact/unstage.sh' script.
|
||||
All stage / unstage operations on the same artifact must be performed from the same working directory.
|
||||
|
||||
Once a stage is complete, 'artifact/upload_stage_packed.sh' can be used to create a packed artifact
|
||||
from the stage. See also 'artifact/upload/pack.sh' and 'artifact/upload/build.sh' for more staging options.
|
||||
|
||||
Example Usage:
|
||||
|
||||
Stage built binaries and .cmake files in \${ARTIFACT_UPLOAD_STAGE}/test_artifacts for upload:
|
||||
$0 test_artifacts 'bin/.*' 'lib/.*' '.*cmake$'
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -lt 2 ]]; then
|
||||
echo "Error: Missing arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
artifact_name="$1"
|
||||
shift
|
||||
regexes=("$@")
|
||||
|
||||
artifact_stage_path="${ARTIFACT_UPLOAD_STAGE}/${artifact_name}"
|
||||
if [[ "$artifact_stage_path" != /* ]]; then
|
||||
artifact_stage_path="$(pwd)/$artifact_stage_path"
|
||||
fi
|
||||
|
||||
mkdir -p "$artifact_stage_path"
|
||||
|
||||
artifact_index_file="$artifact_stage_path/artifact_index.txt"
|
||||
artifact_index_cwd="$artifact_stage_path/artifact_index_cwd.txt"
|
||||
|
||||
if [[ -f "$artifact_index_cwd" ]]; then
|
||||
# Check that the cwd matches the original staging directory if the index already exists:
|
||||
if [[ "$(cat "$artifact_index_cwd")" != "$(pwd)" ]]; then
|
||||
echo "Error: The current working directory has changed since the artifact was staged." >&2
|
||||
echo "Cannot currently stage files from multiple source directories." >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
pwd > "$artifact_index_cwd"
|
||||
fi
|
||||
|
||||
echo "Staging artifacts in '$artifact_stage_path'"
|
||||
for regex in "${regexes[@]}"; do
|
||||
# Prepend './' to the regex for convenience. There's an implied ^ at the start of the find regex,
|
||||
# and paths always start with ./, so this lets us match top level files directly.
|
||||
regex="\\./$regex"
|
||||
echo "Staging files matching regex: $regex"
|
||||
find . -type f -regex "$regex" | tee -a "$artifact_index_file"
|
||||
echo
|
||||
done
|
||||
56
cccl_upstream/ci/util/artifacts/unstage.sh
Executable file
56
cccl_upstream/ci/util/artifacts/unstage.sh
Executable file
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <artifact_name> <regex> [<regex> ...]
|
||||
|
||||
Unstages (removes) files matching the provided regexes from the specified artifact stage.
|
||||
Regexes follow the same rules as artifact/stage.sh.
|
||||
|
||||
This may be used to remove files that were previously staged for upload before packing or building the artifacts.
|
||||
|
||||
Example Usage:
|
||||
|
||||
Unstage previously-staged built binaries and .cmake files from test_artifacts:
|
||||
$0 test_artifacts 'bin/.*' 'lib/.*' '.*cmake$'
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -lt 2 ]]; then
|
||||
echo "Error: Missing arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
artifact_name="$1"
|
||||
shift
|
||||
regexes=("$@")
|
||||
|
||||
artifact_stage_path="${ARTIFACT_UPLOAD_STAGE}/${artifact_name}"
|
||||
if [[ "$artifact_stage_path" != /* ]]; then
|
||||
artifact_stage_path="$(pwd)/$artifact_stage_path"
|
||||
fi
|
||||
|
||||
mkdir -p "$artifact_stage_path"
|
||||
|
||||
artifact_index_file="$artifact_stage_path/artifact_index.txt"
|
||||
artifact_index_cwd="$artifact_stage_path/artifact_index_cwd.txt"
|
||||
|
||||
pwd > "$artifact_index_cwd"
|
||||
|
||||
echo "Unstaging artifacts in '$artifact_stage_path'"
|
||||
for regex in "${regexes[@]}"; do
|
||||
# Modify regex for consistency with staging script:
|
||||
regex="^\\./$regex"
|
||||
echo "Unstaging files matching regex: $regex"
|
||||
grep -E "$regex" "$artifact_index_file"
|
||||
grep -v -E "$regex" "$artifact_index_file" > "${artifact_index_file}.tmp" && \
|
||||
mv "${artifact_index_file}.tmp" "$artifact_index_file"
|
||||
done
|
||||
59
cccl_upstream/ci/util/artifacts/upload.sh
Executable file
59
cccl_upstream/ci/util/artifacts/upload.sh
Executable file
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <name> [<regex> ...]
|
||||
|
||||
Creates an artifact consisting of a zip file containing a single file or set of regex matches.
|
||||
|
||||
Regexes are passed to the $(command -v find) command's -regex option in the current directory.
|
||||
'./' is prepended to all regexes for convenience.
|
||||
The artifact will contain all matching files with paths relative to the current directory.
|
||||
|
||||
If no regexes are provided, the artifact will be created from a file in the current directory.
|
||||
The file must have the same name as the artifact.
|
||||
|
||||
Example Usage:
|
||||
|
||||
Create an artifact of the given file in the current directory using the filename as the artifact name:
|
||||
|
||||
$0 some_resource.log
|
||||
|
||||
Copy all files that match the regexes to a staging directory, and upload a artifact that zips this directory.
|
||||
|
||||
$0 job-\$JOB_ID-products \
|
||||
'build\.ninja$' \
|
||||
'.*rules\.ninja$' \
|
||||
'CMakeCache\.txt$' \
|
||||
'.*VerifyGlobs\.cmake$' \
|
||||
'.*CTestTestfile\.cmake$' \
|
||||
'bin/.*' \
|
||||
'lib/.*'
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -lt 1 ]]; then
|
||||
echo "Error: Missing artifact name." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
readonly artifact_name="$1"
|
||||
|
||||
# If no regexes are provided, use the artifact name as the path:
|
||||
if [[ "$#" -eq 1 ]]; then
|
||||
"$ci_dir/util/artifacts/upload/register.sh" "$artifact_name" "$artifact_name"
|
||||
exit
|
||||
fi
|
||||
|
||||
shift
|
||||
|
||||
"$ci_dir/util/artifacts/stage.sh" "$artifact_name" "$@" > /dev/null
|
||||
"$ci_dir/util/artifacts/upload_stage.sh" "$artifact_name"
|
||||
44
cccl_upstream/ci/util/artifacts/upload/build.sh
Executable file
44
cccl_upstream/ci/util/artifacts/upload/build.sh
Executable file
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <artifact_name>
|
||||
|
||||
Builds a physical tree containing a staged artifact created using artifact/stage.sh / unstage.sh.
|
||||
|
||||
The artifact root will be located at \${ARTIFACT_UPLOAD_STAGE}/<artifact_name>/<artifact_name>.
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -ne 1 ]]; then
|
||||
echo "Error: Invalid number of arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
readonly artifact_name="$1"
|
||||
readonly artifact_stage_path="${ARTIFACT_UPLOAD_STAGE}/${artifact_name}"
|
||||
readonly artifact_index_file="$artifact_stage_path/artifact_index.txt"
|
||||
readonly artifact_cwd_file="$artifact_stage_path/artifact_index_cwd.txt"
|
||||
readonly artifact_dir="${ARTIFACT_UPLOAD_STAGE}/${artifact_name}/${artifact_name}"
|
||||
artifact_cwd="$(cat "$artifact_cwd_file")"
|
||||
readonly artifact_cwd
|
||||
|
||||
mkdir -p "$artifact_dir"
|
||||
|
||||
echo "Building artifact '$artifact_name' in '$artifact_dir'"
|
||||
echo "Pulling artifacts from working directory: $artifact_cwd"
|
||||
|
||||
(
|
||||
cd "$artifact_cwd"
|
||||
while IFS= read -r file; do
|
||||
cp -v --parents "$file" "$artifact_dir"
|
||||
done < "$artifact_index_file"
|
||||
)
|
||||
47
cccl_upstream/ci/util/artifacts/upload/pack.sh
Executable file
47
cccl_upstream/ci/util/artifacts/upload/pack.sh
Executable file
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <artifact_name>
|
||||
|
||||
Packs a staged artifact (created using artifact/stage.sh) into a tar.zst archive.
|
||||
|
||||
The archive will be generated from the staged index file in \${ARTIFACT_UPLOAD_STAGE}/<artifact_name> and
|
||||
saved to \${ARTIFACT_UPLOAD_STAGE}/<artifact_name>/<artifact_name>.tar.zst.
|
||||
|
||||
Example Usages:
|
||||
- $0 test_artifact
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -ne 1 ]]; then
|
||||
echo "Error: Invalid number of arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v zstd > /dev/null 2>&1; then
|
||||
echo "Error: zstd not found." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
readonly artifact_name="$1"
|
||||
readonly artifact_stage_path="${ARTIFACT_UPLOAD_STAGE}/${artifact_name}"
|
||||
readonly artifact_index_file="$artifact_stage_path/artifact_index.txt"
|
||||
readonly artifact_cwd_file="$artifact_stage_path/artifact_index_cwd.txt"
|
||||
readonly artifact_archive="${ARTIFACT_UPLOAD_STAGE}/${artifact_name}/${artifact_name}.tar.zst"
|
||||
|
||||
echo "Packing artifact '$artifact_stage_path' into '$artifact_archive'"
|
||||
echo "Using zstd: $(command -v zstd)"
|
||||
echo "Pulling artifacts from working directory: $(cat "$artifact_cwd_file")"
|
||||
|
||||
tar -cv -C "$(cat "$artifact_cwd_file")" -T "$artifact_index_file" \
|
||||
| zstd --compress --threads=0 \
|
||||
> "$artifact_archive"
|
||||
21
cccl_upstream/ci/util/artifacts/upload/print_matrix.sh
Executable file
21
cccl_upstream/ci/util/artifacts/upload/print_matrix.sh
Executable file
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Prints the Github Actions matrix for uploading all registered artifacts.
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -ne 0 ]]; then
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
jq -c '.' "${ARTIFACT_UPLOAD_REGISTERY:?}"
|
||||
61
cccl_upstream/ci/util/artifacts/upload/register.sh
Executable file
61
cccl_upstream/ci/util/artifacts/upload/register.sh
Executable file
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
readonly artifact_compression_level=6
|
||||
readonly artifact_retention_days=7
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <artifact_name> [<artifact_path>]
|
||||
|
||||
Registers artifacts for upload. If path is not provided, it defaults to the artifact name.
|
||||
|
||||
Compression level is set to $artifact_compression_level by default.
|
||||
Use 'upload/set_compression_level.sh' to change this after registering if needed.
|
||||
|
||||
Default retention days is set to $artifact_retention_days.
|
||||
Use 'upload/set_retention_days.sh' after registering to change this if needed.
|
||||
|
||||
Example Usages:
|
||||
- $0 my_artifact.tar.gz # Assumes the artifact is in the current directory.
|
||||
- $0 my_artifact /path/to/my_artifact.tar.gz
|
||||
- $0 my_artifact /path/to/my_artifact_directory/
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -lt 1 ]]; then
|
||||
echo "Error: Missing artifact name." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
artifact_name="$1"
|
||||
artifact_path="${2:-$artifact_name}"
|
||||
|
||||
# Ensure the artifact path is absolute
|
||||
if [[ "$artifact_path" != /* ]]; then
|
||||
artifact_path="$(pwd)/$artifact_path"
|
||||
fi
|
||||
|
||||
if [[ ! -e "$artifact_path" ]]; then
|
||||
echo "Error: Artifact path '$artifact_path' does not exist." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Register the artifact:
|
||||
jq --arg name "$artifact_name" \
|
||||
--arg path "$artifact_path" \
|
||||
--arg retention_days "$artifact_retention_days" \
|
||||
--argjson compression_level "$artifact_compression_level" \
|
||||
'. += [{"name": $name, "path": $path, "retention_days": ($retention_days | tonumber), "compression_level": $compression_level}]' \
|
||||
"$ARTIFACT_UPLOAD_REGISTERY" > "$ARTIFACT_UPLOAD_REGISTERY.tmp" && \
|
||||
mv "$ARTIFACT_UPLOAD_REGISTERY.tmp" "$ARTIFACT_UPLOAD_REGISTERY"
|
||||
|
||||
echo "Artifact '$artifact_name' registered for upload with path '$artifact_path'."
|
||||
35
cccl_upstream/ci/util/artifacts/upload/set_compression_level.sh
Executable file
35
cccl_upstream/ci/util/artifacts/upload/set_compression_level.sh
Executable file
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <artifact_name> <compression_level>
|
||||
|
||||
Sets the compression level for an artifact registered for upload.
|
||||
|
||||
Example Usage:
|
||||
$0 some_huge_precompressed_archive 0
|
||||
$0 some_many_small_uncompressed_files 10
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -lt 2 ]]; then
|
||||
echo "Error: Missing arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
artifact_name="$1"
|
||||
compression_level="$2"
|
||||
|
||||
# Find the artifact entry and update its compression level
|
||||
jq --arg name "$artifact_name" --argjson compression_level "$compression_level" \
|
||||
'map(if .name == $name then .compression_level = $compression_level else . end)' \
|
||||
"$ARTIFACT_UPLOAD_REGISTERY" > "$ARTIFACT_UPLOAD_REGISTERY.tmp" && \
|
||||
mv "$ARTIFACT_UPLOAD_REGISTERY.tmp" "$ARTIFACT_UPLOAD_REGISTERY"
|
||||
36
cccl_upstream/ci/util/artifacts/upload/set_retention_days.sh
Executable file
36
cccl_upstream/ci/util/artifacts/upload/set_retention_days.sh
Executable file
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <artifact_name> <retention_days>
|
||||
|
||||
Sets the retention days for an artifact registered for upload.
|
||||
|
||||
Example Usage:
|
||||
$0 some_huge_temporary_artifact 1
|
||||
$0 some_small_useful_output 7
|
||||
$0 some_long_term_artifact 30
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -lt 2 ]]; then
|
||||
echo "Error: Missing arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
artifact_name="$1"
|
||||
retention_days="$2"
|
||||
|
||||
# Find the artifact entry and update its retention days
|
||||
jq --arg name "$artifact_name" --argjson retention_days "$retention_days" \
|
||||
'map(if .name == $name then .retention_days = $retention_days else . end)' \
|
||||
"$ARTIFACT_UPLOAD_REGISTERY" > "$ARTIFACT_UPLOAD_REGISTERY.tmp" && \
|
||||
mv "$ARTIFACT_UPLOAD_REGISTERY.tmp" "$ARTIFACT_UPLOAD_REGISTERY"
|
||||
54
cccl_upstream/ci/util/artifacts/upload_packed.sh
Executable file
54
cccl_upstream/ci/util/artifacts/upload_packed.sh
Executable file
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <name> <regex> [<regex> ...]
|
||||
|
||||
Create a compressed artifact, suitable for large, temporary files such as build products or test binaries
|
||||
that need to be quickly uploaded and downloaded between CI jobs. The artifact will exist of a
|
||||
zip file containing an <artifact_name>.tar.zst archive, packed with the parallel zstd.
|
||||
|
||||
Regexes are passed to the $(command -v find) command's -regex option in the current directory.
|
||||
'./' is prepended to all regexes for convenience.
|
||||
The artifact will contain all matching files relative to the current directory.
|
||||
|
||||
If no regexes are provided, the artifact will be created from a file in the current directory.
|
||||
The file must have the same name as the artifact.
|
||||
|
||||
Example Usage:
|
||||
|
||||
Create an artifact of the given file in the current directory using the filename as the artifact name:
|
||||
|
||||
$0 some_resource.log
|
||||
|
||||
Copy all files that match the regexes to a staging directory, and upload a artifact that zips this directory.
|
||||
|
||||
$0 job-\$JOB_ID-products \
|
||||
'build\.ninja$' \
|
||||
'.*rules\.ninja$' \
|
||||
'CMakeCache\.txt$' \
|
||||
'.*VerifyGlobs\.cmake$' \
|
||||
'.*CTestTestfile\.cmake$' \
|
||||
'bin/.*' \
|
||||
'lib/.*'
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -lt 2 ]]; then
|
||||
echo "Error: Invalid number of arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
readonly artifact_name="$1"
|
||||
shift
|
||||
|
||||
"$ci_dir/util/artifacts/stage.sh" "$artifact_name" "$@" > /dev/null
|
||||
"$ci_dir/util/artifacts/upload_stage_packed.sh" "$artifact_name"
|
||||
31
cccl_upstream/ci/util/artifacts/upload_stage.sh
Executable file
31
cccl_upstream/ci/util/artifacts/upload_stage.sh
Executable file
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <name>
|
||||
|
||||
Same as 'ci/util/artifacts/upload_packed.sh', but assumes that the stage has already been created using
|
||||
'ci/util/artifacts/stage.sh' and 'unstage.sh'. Performs the packing and registration steps only.
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -ne 1 ]]; then
|
||||
echo "Error: Invalid number of arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
readonly artifact_name="$1"
|
||||
readonly artifact_dir="$ARTIFACT_UPLOAD_STAGE/${artifact_name}/${artifact_name}"
|
||||
|
||||
start=$SECONDS
|
||||
"$ci_dir/util/artifacts/upload/build.sh" "$artifact_name"
|
||||
"$ci_dir/util/artifacts/upload/register.sh" "$artifact_name" "$artifact_dir"
|
||||
echo "Artifact '$artifact_name' built in $((SECONDS - start)) seconds."
|
||||
34
cccl_upstream/ci/util/artifacts/upload_stage_packed.sh
Executable file
34
cccl_upstream/ci/util/artifacts/upload_stage_packed.sh
Executable file
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/artifacts/common.sh
|
||||
source "$ci_dir/util/artifacts/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 <name>
|
||||
|
||||
Same as 'ci/util/artifacts/upload_packed.sh', but assumes that the stage has already been created using
|
||||
'ci/util/artifacts/stage.sh' and 'unstage.sh'. Performs the packing and registration steps only.
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -ne 1 ]]; then
|
||||
echo "Error: Invalid number of arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
readonly artifact_name="$1"
|
||||
# shellcheck disable=SC2154
|
||||
readonly artifact_archive="$ARTIFACT_UPLOAD_STAGE/${artifact_name}/${artifact_name}.tar.zst"
|
||||
|
||||
start=$SECONDS
|
||||
"$ci_dir/util/artifacts/upload/pack.sh" "$artifact_name"
|
||||
"$ci_dir/util/artifacts/upload/register.sh" "$artifact_name" "$artifact_archive"
|
||||
# Already compressed while packing:
|
||||
"$ci_dir/util/artifacts/upload/set_compression_level.sh" "$artifact_name" 0 > /dev/null
|
||||
echo "Artifact '$artifact_name' packed in $((SECONDS - start)) seconds."
|
||||
Reference in New Issue
Block a user