feat(cccl): integrate missing CCCL directories — python/, ci/, .agent/, docs/, test/
Sparse-checkout from NVIDIA/cccl main branch to complete cccl_upstream: Added: - python/cuda_cccl/ (226 files) — Python bindings for device-level algorithms Critical for muh toolchain: cuda.compute.reduce_into, scan, radix_sort, etc. Includes 204 .py files with full test coverage for all 27 algorithms - ci/ (163 files) — Build/test infrastructure build_cub.sh, test_cub.sh, build_and_test_targets.sh, matrix.yaml Directly maps to our [INFRA-CI] and [INFRA-BUILD] items - .agent/skills/ (7 files) — NVIDIA's own agent skills for CCCL cccl-style/SKILL.md, cccl-test/SKILL.md, sass-diff/SKILL.md - docs/ (491 files) — Official CCCL documentation CI references, CMake guides, Python compute docs, libcudacxx PTX docs - test/ (12 files) — Top-level integration tests (cuda_smoke, stdpar) - Root configs: .clang-format, .clang-tidy, CONTRIBUTING.md, pyproject.toml - CLAUDE.md symlink → AGENTS.md (NVIDIA's standard) cccl_upstream now mirrors full NVIDIA/cccl structure: Before: 42M (cub + thrust + libcudacxx + cudax + c + examples + benchmarks) After: 53M (+python +ci +docs +.agent +test +configs) This completes the CCCL base needed for: - [muh-bench] items: ci/util/build_and_test_targets.sh for targeted builds - [CCCL-verify] items: python/cuda_cccl/tests/ as reference implementations - [CCCL-test] items: ci/test_cub.sh, ci/test_thrust.sh - Agent workflow: .agent/skills/ for consistent style and test patterns
This commit is contained in:
muh-bot
34
cccl_upstream/ci/util/workflow/common.sh
Executable file
34
cccl_upstream/ci/util/workflow/common.sh
Executable file
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
||||
echo "This script must be sourced, not executed directly." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${GITHUB_ACTIONS:-}" ]]; then
|
||||
echo "This script must be run in a GitHub Actions environment." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
to_posix_path() {
|
||||
local path="$1"
|
||||
|
||||
if [[ "$path" =~ ^([A-Za-z]):([\\/]?.*)$ ]]; then
|
||||
local drive="${BASH_REMATCH[1]}"
|
||||
local rest="${BASH_REMATCH[2]}"
|
||||
rest="${rest//\\/\/}"
|
||||
printf '/%s%s\n' "${drive,,}" "$rest"
|
||||
return
|
||||
fi
|
||||
|
||||
printf '%s\n' "$path"
|
||||
}
|
||||
|
||||
runner_temp_posix="$(to_posix_path "${RUNNER_TEMP:-/tmp}")"
|
||||
|
||||
export WORKFLOW_ARTIFACT="workflow"
|
||||
export WORKFLOW_DIR="${runner_temp_posix}/workflow"
|
||||
|
||||
mkdir -p "$WORKFLOW_DIR"
|
||||
43
cccl_upstream/ci/util/workflow/get_consumers.sh
Executable file
43
cccl_upstream/ci/util/workflow/get_consumers.sh
Executable file
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/workflow/common.sh
|
||||
source "$ci_dir/util/workflow/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 [job_id]
|
||||
|
||||
Return a json array of job definitions for all consumers of the specified producer job ID.
|
||||
If no job ID is provided, the \$JOB_ID environment variable is used.
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -gt 1 ]]; then
|
||||
echo "Error: Too many arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
job_id="${1:-${JOB_ID:-}}"
|
||||
|
||||
if [[ -z "$job_id" ]]; then
|
||||
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"${ci_dir}/util/workflow/initialize.sh"
|
||||
|
||||
consumers=$(jq --arg job_id "$job_id" '
|
||||
to_entries[]
|
||||
| select(.value.two_stage)
|
||||
| .value.two_stage[]
|
||||
| select(any(.producers[]; .id == $job_id))
|
||||
| .consumers
|
||||
' "$WORKFLOW_DIR/workflow.json")
|
||||
|
||||
echo "$consumers"
|
||||
51
cccl_upstream/ci/util/workflow/get_job_def.sh
Executable file
51
cccl_upstream/ci/util/workflow/get_job_def.sh
Executable file
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/workflow/common.sh
|
||||
source "$ci_dir/util/workflow/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 [job_id]
|
||||
|
||||
Prints a json object containing the workflow job definition for the specified job ID.
|
||||
If no job ID is provided, the \$JOB_ID environment variable is used.
|
||||
If the job ID does not exist in the workflow, an error is raised.
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -gt 1 ]]; then
|
||||
echo "Error: Too many arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
job_id="${1:-${JOB_ID:-}}"
|
||||
|
||||
if [[ -z "$job_id" ]]; then
|
||||
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"${ci_dir}/util/workflow/initialize.sh"
|
||||
|
||||
job_obj=$(jq --arg job_id "$job_id" '
|
||||
to_entries[]
|
||||
| .value
|
||||
| (
|
||||
(select(has("standalone")) | .standalone[] | select(.id == $job_id)) //
|
||||
(select(has("two_stage")) | .two_stage[] | .producers[] | select(.id == $job_id)) //
|
||||
(select(has("two_stage")) | .two_stage[] | .consumers[] | select(.id == $job_id))
|
||||
)
|
||||
' "$WORKFLOW_DIR/workflow.json")
|
||||
|
||||
if [[ -z "$job_obj" ]]; then
|
||||
echo "Error: No job definition found for job ID '$job_id'." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "$job_obj" | jq -r
|
||||
34
cccl_upstream/ci/util/workflow/get_job_project.sh
Executable file
34
cccl_upstream/ci/util/workflow/get_job_project.sh
Executable file
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 [job_id]
|
||||
|
||||
Returns the name of the project built by the specified job id.
|
||||
If no job ID is provided, the \$JOB_ID environment variable is used.
|
||||
If the job ID does not exist in the workflow an error is raised.
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -gt 1 ]]; then
|
||||
echo "Error: Too many arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
job_id="${1:-${JOB_ID:-}}"
|
||||
|
||||
if [[ -z "$job_id" ]]; then
|
||||
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
job_def=$("${ci_dir}/util/workflow/get_job_def.sh" "$job_id")
|
||||
project=$(echo "$job_def" | jq -r '.origin.matrix_job.project')
|
||||
echo "$project"
|
||||
55
cccl_upstream/ci/util/workflow/get_producer_id.sh
Executable file
55
cccl_upstream/ci/util/workflow/get_producer_id.sh
Executable file
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/workflow/common.sh
|
||||
source "$ci_dir/util/workflow/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 [job_id]
|
||||
|
||||
Prints the job ID of the associated producer for the specified consumer job ID.
|
||||
If no job ID is provided, the \$JOB_ID environment variable is used.
|
||||
If the number of producers for the job is not exactly one, an error is raised.
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -gt 1 ]]; then
|
||||
echo "Error: Too many arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
job_id="${1:-${JOB_ID:-}}"
|
||||
|
||||
if [[ -z "$job_id" ]]; then
|
||||
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"${ci_dir}/util/workflow/initialize.sh"
|
||||
|
||||
producers=$(jq --arg job_id "$job_id" '
|
||||
to_entries[]
|
||||
| select(.value.two_stage)
|
||||
| .value.two_stage[]
|
||||
| select(any(.consumers[]; .id == $job_id))
|
||||
| .producers
|
||||
' "$WORKFLOW_DIR/workflow.json")
|
||||
|
||||
producer_count=$(echo "$producers" | jq 'length')
|
||||
if [[ "$producer_count" -ne 1 ]]; then
|
||||
echo "Error: Expected exactly one producer for job ID '$job_id', but found ${producer_count:-0}." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
producer_id=$(echo "$producers" | jq -r '.[0].id')
|
||||
if [[ -z "$producer_id" ]]; then
|
||||
echo "Error: No producer ID found for job ID '$job_id'." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$producer_id"
|
||||
43
cccl_upstream/ci/util/workflow/get_producers.sh
Executable file
43
cccl_upstream/ci/util/workflow/get_producers.sh
Executable file
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/workflow/common.sh
|
||||
source "$ci_dir/util/workflow/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 [job_id]
|
||||
|
||||
Return a json array of job definitions for all producers of the specified consumer job ID.
|
||||
If no job ID is provided, the \$JOB_ID environment variable is used.
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -gt 1 ]]; then
|
||||
echo "Error: Too many arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
job_id="${1:-${JOB_ID:-}}"
|
||||
|
||||
if [[ -z "$job_id" ]]; then
|
||||
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"${ci_dir}/util/workflow/initialize.sh"
|
||||
|
||||
producers=$(jq --arg job_id "$job_id" '
|
||||
to_entries[]
|
||||
| select(.value.two_stage)
|
||||
| .value.two_stage[]
|
||||
| select(any(.consumers[]; .id == $job_id))
|
||||
| .producers
|
||||
' "$WORKFLOW_DIR/workflow.json")
|
||||
|
||||
echo "$producers"
|
||||
35
cccl_upstream/ci/util/workflow/get_stable_job_hash.sh
Executable file
35
cccl_upstream/ci/util/workflow/get_stable_job_hash.sh
Executable file
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 [job_id]
|
||||
|
||||
Get a stable hash that identifies the job's toolchain, runner, image,
|
||||
name, and launch command, removing origin and per-run ids.
|
||||
If no job ID is provided, the \$JOB_ID environment variable is used.
|
||||
If the job ID does not exist in the workflow an error is raised.
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -gt 1 ]]; then
|
||||
echo "Error: Too many arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
job_id="${1:-${JOB_ID:-}}"
|
||||
|
||||
if [[ -z "$job_id" ]]; then
|
||||
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
job_def=$("${ci_dir}/util/workflow/get_job_def.sh" "$job_id" | jq 'del(.id, .origin)')
|
||||
job_hash=$(echo "$job_def" | sha256sum | awk '{print $1}')
|
||||
echo "$job_hash"
|
||||
68
cccl_upstream/ci/util/workflow/get_wheel_artifact_name.sh
Executable file
68
cccl_upstream/ci/util/workflow/get_wheel_artifact_name.sh
Executable file
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 [job_id]
|
||||
|
||||
Get the name of the wheel file that matches the specified job ID's configuration.
|
||||
If no job ID is provided, the \$JOB_ID environment variable is used.
|
||||
If the job ID does not exist in the workflow, or is not a python job, an error is raised.
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -gt 1 ]]; then
|
||||
echo "Error: Too many arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
job_id="${1:-${JOB_ID:-}}"
|
||||
|
||||
if [[ -z "$job_id" ]]; then
|
||||
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
job_def=$("${ci_dir}/util/workflow/get_job_def.sh" "$job_id")
|
||||
|
||||
py_version=$(echo "$job_def" | jq -r '.origin.matrix_job.py_version')
|
||||
host=$(echo "$job_def" | jq -r '.origin.matrix_job.cxx_family')
|
||||
if [[ "$host" == "MSVC" ]]; then
|
||||
os="windows"
|
||||
else
|
||||
os="linux"
|
||||
fi
|
||||
arch=$(echo "$job_def" | jq -r '.origin.matrix_job.cpu')
|
||||
project=$(echo "$job_def" | jq -r '.origin.matrix_job.project')
|
||||
|
||||
for tag in "$py_version" "$os" "$arch"; do
|
||||
if [[ -z "$tag" ]]; then
|
||||
echo "Error: Missing required field in job definition for job ID '$job_id'." >&2
|
||||
echo "$usage" >&2
|
||||
echo >&2
|
||||
"Job definition: $job_def" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# v1 and v2 Python build jobs both run in the same workflow, so their wheel
|
||||
# artifacts must have distinct names or the second upload clobbers the first
|
||||
# and downstream test jobs grab the wrong wheel. v1 keeps its historical name
|
||||
# (the test-cpu-import workflow hardcodes it); v2 gets a "-v2" suffix.
|
||||
suffix=""
|
||||
if [[ "$project" == "python_v2" ]]; then
|
||||
suffix="-v2"
|
||||
elif [[ "$project" == "python_tsan" ]]; then
|
||||
# ThreadSanitizer-instrumented wheel (free-threaded TSan nightly lane). Must
|
||||
# be distinct so its build doesn't clobber the normal wheel and the TSan test
|
||||
# job doesn't grab an uninstrumented one.
|
||||
suffix="-tsan"
|
||||
fi
|
||||
|
||||
echo "wheel-cccl${suffix}-$os-$arch-py$py_version"
|
||||
47
cccl_upstream/ci/util/workflow/has_consumers.sh
Executable file
47
cccl_upstream/ci/util/workflow/has_consumers.sh
Executable file
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/workflow/common.sh
|
||||
source "$ci_dir/util/workflow/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 [job_id]
|
||||
|
||||
Exits successfully if the specified job ID has consumers, otherwise exits with an error.
|
||||
If no job ID is provided, the \$JOB_ID environment variable is used.
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -gt 1 ]]; then
|
||||
echo "Error: Too many arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
job_id="${1:-${JOB_ID:-}}"
|
||||
|
||||
if [[ -z "$job_id" ]]; then
|
||||
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"${ci_dir}/util/workflow/initialize.sh"
|
||||
|
||||
matching_producer=$(jq --arg job_id "$job_id" '
|
||||
to_entries[]
|
||||
| select(.value.two_stage)
|
||||
| .value.two_stage[]
|
||||
| .producers[]
|
||||
| select(.id == $job_id)
|
||||
' "$WORKFLOW_DIR/workflow.json")
|
||||
|
||||
if [[ -n "$matching_producer" ]]; then
|
||||
exit 0
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
47
cccl_upstream/ci/util/workflow/has_producers.sh
Executable file
47
cccl_upstream/ci/util/workflow/has_producers.sh
Executable file
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/workflow/common.sh
|
||||
source "$ci_dir/util/workflow/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0 [job_id]
|
||||
|
||||
Exits successfully if the specified job ID has producers, otherwise exits with an error.
|
||||
If no job ID is provided, the \$JOB_ID environment variable is used.
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -gt 1 ]]; then
|
||||
echo "Error: Too many arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
job_id="${1:-${JOB_ID:-}}"
|
||||
|
||||
if [[ -z "$job_id" ]]; then
|
||||
echo "Error: No job ID provided and \$JOB_ID is not set." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"${ci_dir}/util/workflow/initialize.sh"
|
||||
|
||||
matching_consumer=$(jq --arg job_id "$job_id" '
|
||||
to_entries[]
|
||||
| select(.value.two_stage)
|
||||
| .value.two_stage[]
|
||||
| .consumers[]
|
||||
| select(.id == $job_id)
|
||||
' "$WORKFLOW_DIR/workflow.json")
|
||||
|
||||
if [[ -n "$matching_consumer" ]]; then
|
||||
exit 0
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
26
cccl_upstream/ci/util/workflow/initialize.sh
Executable file
26
cccl_upstream/ci/util/workflow/initialize.sh
Executable file
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
|
||||
readonly ci_dir
|
||||
# shellcheck source=ci/util/workflow/common.sh
|
||||
source "$ci_dir/util/workflow/common.sh"
|
||||
|
||||
usage=$(cat <<EOF
|
||||
Usage: $0
|
||||
|
||||
Downloads the workflow artifact and unpacks it to \$WORKFLOW_DIR, but only if it doesn't already exist.
|
||||
EOF
|
||||
)
|
||||
readonly usage
|
||||
|
||||
if [[ "$#" -ne 0 ]]; then
|
||||
echo "Error: This script does not take any arguments." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$WORKFLOW_DIR/workflow.json" ]]; then
|
||||
"$ci_dir/util/artifacts/download/fetch.sh" "$WORKFLOW_ARTIFACT" "$WORKFLOW_DIR" > /dev/null
|
||||
fi
|
||||
Reference in New Issue
Block a user