feat(cccl): integrate missing CCCL directories — python/, ci/, .agent/, docs/, test/
Sparse-checkout from NVIDIA/cccl main branch to complete cccl_upstream: Added: - python/cuda_cccl/ (226 files) — Python bindings for device-level algorithms Critical for muh toolchain: cuda.compute.reduce_into, scan, radix_sort, etc. Includes 204 .py files with full test coverage for all 27 algorithms - ci/ (163 files) — Build/test infrastructure build_cub.sh, test_cub.sh, build_and_test_targets.sh, matrix.yaml Directly maps to our [INFRA-CI] and [INFRA-BUILD] items - .agent/skills/ (7 files) — NVIDIA's own agent skills for CCCL cccl-style/SKILL.md, cccl-test/SKILL.md, sass-diff/SKILL.md - docs/ (491 files) — Official CCCL documentation CI references, CMake guides, Python compute docs, libcudacxx PTX docs - test/ (12 files) — Top-level integration tests (cuda_smoke, stdpar) - Root configs: .clang-format, .clang-tidy, CONTRIBUTING.md, pyproject.toml - CLAUDE.md symlink → AGENTS.md (NVIDIA's standard) cccl_upstream now mirrors full NVIDIA/cccl structure: Before: 42M (cub + thrust + libcudacxx + cudax + c + examples + benchmarks) After: 53M (+python +ci +docs +.agent +test +configs) This completes the CCCL base needed for: - [muh-bench] items: ci/util/build_and_test_targets.sh for targeted builds - [CCCL-verify] items: python/cuda_cccl/tests/ as reference implementations - [CCCL-test] items: ci/test_cub.sh, ci/test_thrust.sh - Agent workflow: .agent/skills/ for consistent style and test patterns
This commit is contained in:
muh-bot
139
cccl_upstream/ci/util/pre-commit/check_cub_test_macros.py
Executable file
139
cccl_upstream/ci/util/pre-commit/check_cub_test_macros.py
Executable file
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
|
||||
import re
|
||||
import sys
|
||||
|
||||
RAW_TEST_MACROS = (
|
||||
"C2H_TEST",
|
||||
"C2H_TEST_LIST",
|
||||
"C2H_TEST_WITH_FIXTURE",
|
||||
"C2H_TEST_LIST_WITH_FIXTURE",
|
||||
"TEST_CASE",
|
||||
"TEST_CASE_METHOD",
|
||||
"SCENARIO",
|
||||
"SCENARIO_METHOD",
|
||||
"TEMPLATE_TEST_CASE",
|
||||
"TEMPLATE_TEST_CASE_SIG",
|
||||
"TEMPLATE_TEST_CASE_METHOD",
|
||||
"TEMPLATE_TEST_CASE_METHOD_SIG",
|
||||
"TEMPLATE_PRODUCT_TEST_CASE",
|
||||
"TEMPLATE_PRODUCT_TEST_CASE_SIG",
|
||||
"TEMPLATE_PRODUCT_TEST_CASE_METHOD",
|
||||
"TEMPLATE_PRODUCT_TEST_CASE_METHOD_SIG",
|
||||
"TEMPLATE_LIST_TEST_CASE",
|
||||
"TEMPLATE_LIST_TEST_CASE_METHOD",
|
||||
)
|
||||
|
||||
RAW_TEST_MACRO_RE = re.compile(
|
||||
r"^[ \t]*(?P<macro>"
|
||||
+ "|".join(
|
||||
re.escape(macro) for macro in sorted(RAW_TEST_MACROS, key=len, reverse=True)
|
||||
)
|
||||
+ r")[ \t]*\(",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def remove_comments(source: str) -> str:
|
||||
"""Blank C++ comments while preserving line and column positions."""
|
||||
result = list(source)
|
||||
index = 0
|
||||
|
||||
while index < len(source):
|
||||
if source.startswith("//", index):
|
||||
end = source.find("\n", index)
|
||||
if end == -1:
|
||||
end = len(source)
|
||||
for comment_index in range(index, end):
|
||||
result[comment_index] = " "
|
||||
index = end
|
||||
continue
|
||||
|
||||
if source.startswith("/*", index):
|
||||
end = source.find("*/", index + 2)
|
||||
if end == -1:
|
||||
end = len(source) - 2
|
||||
for comment_index in range(index, min(end + 2, len(source))):
|
||||
if source[comment_index] not in "\r\n":
|
||||
result[comment_index] = " "
|
||||
index = end + 2
|
||||
continue
|
||||
|
||||
if source[index] in {'"', "'"}:
|
||||
quote = source[index]
|
||||
index += 1
|
||||
while index < len(source):
|
||||
if source[index] == "\\":
|
||||
index += 2
|
||||
continue
|
||||
if source[index] == quote:
|
||||
index += 1
|
||||
break
|
||||
index += 1
|
||||
continue
|
||||
|
||||
index += 1
|
||||
|
||||
return "".join(result)
|
||||
|
||||
|
||||
def self_test() -> bool:
|
||||
fixtures = [
|
||||
('// C2H_TEST("x")', False),
|
||||
('/*\nTEST_CASE("x")\n*/', False),
|
||||
('const char* value = "TEST_CASE(";', False),
|
||||
('CUB_TEST("x", "[y]", CUB_SMALL)', False),
|
||||
('CUB_TEST_CASE("x", "[y]", CUB_LARGE)', False),
|
||||
('CUB_TEST_LIST("x", "[y]", CUB_SMALL, types)', False),
|
||||
]
|
||||
fixtures.extend((f'{macro}("x")', True) for macro in RAW_TEST_MACROS)
|
||||
|
||||
for source, expected in fixtures:
|
||||
found = bool(RAW_TEST_MACRO_RE.search(remove_comments(source)))
|
||||
if found != expected:
|
||||
expected_result = "match" if expected else "no match"
|
||||
actual_result = "match" if found else "no match"
|
||||
print(
|
||||
"internal error: test-registration checker self-test failed for "
|
||||
f"{source!r}: expected {expected_result}, found {actual_result}. "
|
||||
"This is a problem with the checker, not the files being committed.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def check_file(filename: str) -> bool:
|
||||
with open(filename, encoding="utf-8", errors="surrogateescape") as source_file:
|
||||
source = source_file.read()
|
||||
|
||||
source_without_comments = remove_comments(source)
|
||||
found_error = False
|
||||
for match in RAW_TEST_MACRO_RE.finditer(source_without_comments):
|
||||
line = source.count("\n", 0, match.start()) + 1
|
||||
column = match.start("macro") - source.rfind("\n", 0, match.start("macro"))
|
||||
print(
|
||||
f"{filename}:{line}:{column}: {match.group('macro')} bypasses CUB "
|
||||
"memory classification; use CUB_TEST, CUB_TEST_CASE, or CUB_TEST_LIST."
|
||||
)
|
||||
found_error = True
|
||||
|
||||
return found_error
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not self_test():
|
||||
return 2
|
||||
|
||||
found_error = False
|
||||
for filename in sys.argv[1:]:
|
||||
found_error = check_file(filename) or found_error
|
||||
return int(found_error)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
53
cccl_upstream/ci/util/pre-commit/check_shebang.py
Executable file
53
cccl_upstream/ci/util/pre-commit/check_shebang.py
Executable file
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# We pre-compile this regular expression as a micro-optimization for speed. The
|
||||
# assumption is that the shebang is correct, so everything in the happy path should be
|
||||
# as fast as possible.
|
||||
FIRST_LINE_RE = re.compile(r"#!\s*/usr/bin/env.*")
|
||||
ret = 0
|
||||
for f in sys.argv[1:]:
|
||||
with open(f) as fd:
|
||||
first_line = fd.readline()
|
||||
|
||||
if not first_line.startswith("#!"):
|
||||
# Not a shebang
|
||||
continue
|
||||
|
||||
if FIRST_LINE_RE.match(first_line):
|
||||
# Already correct
|
||||
continue
|
||||
|
||||
ret = 1
|
||||
|
||||
if not (
|
||||
m := re.match(r"#!\s*(?:/bin/(\w+)|/usr/bin/(\w+))\s*(.*)", first_line)
|
||||
):
|
||||
# Not assert, pre-commit may compile with -O
|
||||
raise AssertionError(f"Failed to match shebang for {first_line}")
|
||||
|
||||
fixed = f"#!/usr/bin/env {m[1] or m[2]}".rstrip()
|
||||
if rest := m[3].strip():
|
||||
fixed += f" {rest}"
|
||||
fixed += "\n"
|
||||
|
||||
with open(f) as fd:
|
||||
# Read the remaining lines, we need them in order to overwrite
|
||||
lines = fd.readlines()
|
||||
|
||||
lines[0] = fixed
|
||||
|
||||
with open(f, "w") as fd:
|
||||
fd.writelines(lines)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
94
cccl_upstream/ci/util/pre-commit/strip_unprintable.py
Executable file
94
cccl_upstream/ci/util/pre-commit/strip_unprintable.py
Executable file
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
#
|
||||
# strip_unprintable.py - Remove invisible / unprintable characters from text files.
|
||||
import re
|
||||
import sys
|
||||
from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter
|
||||
|
||||
# Canonical definition of what gets removed. One row per range group:
|
||||
# (regex character-class fragment, human description)
|
||||
# The compiled character class and the --help listing are both derived from this
|
||||
# table, so adding or removing a range only needs to happen here. TAB (U+0009),
|
||||
# LF (U+000A), and CR (U+000D) are deliberately excluded from the C0 range. The
|
||||
# fragments use \x/\u escapes so the source file itself stays free of the very
|
||||
# characters this script removes.
|
||||
RANGES = (
|
||||
(
|
||||
r"\x00-\x08\x0b\x0c\x0e-\x1f\x7f",
|
||||
"C0 controls / DEL (TAB, LF, CR preserved)",
|
||||
),
|
||||
(r"\x80-\x9f", "C1 controls"),
|
||||
(r"\xa0", "no-break space"),
|
||||
(r"\u200b-\u200f", "zero-width space/joiners, bidi marks"),
|
||||
(r"\u202a-\u202e", "bidi embedding/override"),
|
||||
(r"\u2060-\u2064", "word joiner, invisible operators"),
|
||||
(r"\ufeff", "BOM / zero-width no-break space"),
|
||||
)
|
||||
|
||||
# Character class assembled from column 1 of the ranges table.
|
||||
BAD_RE = re.compile("[" + "".join(frag for frag, _ in RANGES) + "]")
|
||||
|
||||
|
||||
def parse_args() -> Namespace:
|
||||
removed = "\n".join(f" {frag}\t{desc}" for frag, desc in RANGES)
|
||||
parser = ArgumentParser(
|
||||
description=(
|
||||
"Remove invisible / unprintable characters from text files, in place, "
|
||||
"while preserving ordinary whitespace (TAB U+0009, LF U+000A, "
|
||||
"CR U+000D)."
|
||||
),
|
||||
epilog=f"Removed characters:\n{removed}",
|
||||
formatter_class=RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--check",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Report offending files and their line/column locations; make no "
|
||||
"edits. Exits non-zero if any unprintable characters are found."
|
||||
),
|
||||
)
|
||||
parser.add_argument("files", nargs="+", metavar="FILE")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def check(files: list[str]) -> int:
|
||||
ret = 0
|
||||
for f in files:
|
||||
with open(f, encoding="utf-8", errors="surrogateescape") as fd:
|
||||
for lineno, line in enumerate(fd, start=1):
|
||||
for m in BAD_RE.finditer(line):
|
||||
print(f"{f}:{lineno}:{m.start() + 1}: U+{ord(m.group()):04X}")
|
||||
ret = 1
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def strip(files: list[str]) -> int:
|
||||
ret = 0
|
||||
for f in files:
|
||||
with open(f, encoding="utf-8", errors="surrogateescape") as fd:
|
||||
original = fd.read()
|
||||
stripped = BAD_RE.sub("", original)
|
||||
if stripped != original:
|
||||
with open(
|
||||
f, "w", encoding="utf-8", errors="surrogateescape", newline=""
|
||||
) as fd:
|
||||
fd.write(stripped)
|
||||
ret = 1
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
if args.check:
|
||||
return check(args.files)
|
||||
return strip(args.files)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user